Commit e66584e2 authored by luis_pereira87's avatar luis_pereira87

OAI T2 integration

parent 914b4703
...@@ -608,6 +608,19 @@ if (ENABLE_LDPC_T1) ...@@ -608,6 +608,19 @@ if (ENABLE_LDPC_T1)
target_link_libraries(ldpc_t1 ${LIBDPDK_T1_LDFLAGS} ${T1}) target_link_libraries(ldpc_t1 ${LIBDPDK_T1_LDFLAGS} ${T1})
endif() endif()
add_boolean_option(ENABLE_LDPC_T2 ON "Build support for LDPC Offload to T2 library")
if (ENABLE_LDPC_T2)
pkg_check_modules(LIBDPDK_T2 REQUIRED libdpdk=20.11.3)
find_library(T2 NAMES rte_baseband_accl_ldpc REQUIRED)
if (NOT T2)
message(FATAL_ERROR "Library rte_baseband_accl_ldpc for T2 offload not found")
endif()
add_library(ldpc_t2 MODULE ${OPENAIR1_DIR}/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c)
set_target_properties(ldpc_t2 PROPERTIES COMPILE_FLAGS "-DALLOW_EXPERIMENTAL_API")
target_compile_options(ldpc_t2 PRIVATE ${LIBDPDK_T2_CFLAGS})
target_link_libraries(ldpc_t2 ${LIBDPDK_T2_LDFLAGS} ${T2})
endif()
########################################################## ##########################################################
include_directories ("${OPENAIR_DIR}/radio/COMMON") include_directories ("${OPENAIR_DIR}/radio/COMMON")
...@@ -2370,6 +2383,10 @@ if (ENABLE_LDPC_T1) ...@@ -2370,6 +2383,10 @@ if (ENABLE_LDPC_T1)
add_dependencies(nr-softmodem ldpc_t1) add_dependencies(nr-softmodem ldpc_t1)
endif() endif()
if (ENABLE_LDPC_T2)
add_dependencies(nr-softmodem ldpc_t2)
endif()
# force the generation of ASN.1 so that we don't need to wait during the build # force the generation of ASN.1 so that we don't need to wait during the build
target_link_libraries(nr-softmodem PRIVATE target_link_libraries(nr-softmodem PRIVATE
asn1_lte_rrc asn1_nr_rrc asn1_s1ap asn1_ngap asn1_m2ap asn1_m3ap asn1_x2ap asn1_f1ap asn1_lpp) asn1_lte_rrc asn1_nr_rrc asn1_s1ap asn1_ngap asn1_m2ap asn1_m3ap asn1_x2ap asn1_f1ap asn1_lpp)
...@@ -2618,6 +2635,10 @@ if (ENABLE_LDPC_T1) ...@@ -2618,6 +2635,10 @@ if (ENABLE_LDPC_T1)
add_dependencies(nr_ulsim ldpc_t1) add_dependencies(nr_ulsim ldpc_t1)
endif() endif()
if (ENABLE_LDPC_T2)
add_dependencies(nr_ulsim ldpc_t2)
endif()
target_link_libraries(nr_ulsim PRIVATE target_link_libraries(nr_ulsim PRIVATE
-Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR_COMMON PHY_NR PHY_NR_UE SCHED_NR_LIB SCHED_NR_UE_LIB MAC_NR MAC_UE_NR MAC_NR_COMMON lte_rrc nr_rrc CONFIG_LIB L2_LTE_NR L2_NR HASHTABLE x2ap SECU_CN ngap NFAPI_COMMON_LIB NFAPI_LIB NFAPI_PNF_LIB NFAPI_USER_LIB -lz -Wl,--end-group -Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR_COMMON PHY_NR PHY_NR_UE SCHED_NR_LIB SCHED_NR_UE_LIB MAC_NR MAC_UE_NR MAC_NR_COMMON lte_rrc nr_rrc CONFIG_LIB L2_LTE_NR L2_NR HASHTABLE x2ap SECU_CN ngap NFAPI_COMMON_LIB NFAPI_LIB NFAPI_PNF_LIB NFAPI_USER_LIB -lz -Wl,--end-group
m pthread ${ATLAS_LIBRARIES} ${T_LIB} ITTI ${OPENSSL_LIBRARIES} dl m pthread ${ATLAS_LIBRARIES} ${T_LIB} ITTI ${OPENSSL_LIBRARIES} dl
......
...@@ -28,7 +28,7 @@ ...@@ -28,7 +28,7 @@
{"D" , CONFIG_HLP_DLBM_PHYTEST,0, u64ptr:&dlsch_slot_bitmap, defintval:0, TYPE_UINT64, 0}, \ {"D" , CONFIG_HLP_DLBM_PHYTEST,0, u64ptr:&dlsch_slot_bitmap, defintval:0, TYPE_UINT64, 0}, \
{"U" , CONFIG_HLP_ULBM_PHYTEST,0, u64ptr:&ulsch_slot_bitmap, defintval:0, TYPE_UINT64, 0}, \ {"U" , CONFIG_HLP_ULBM_PHYTEST,0, u64ptr:&ulsch_slot_bitmap, defintval:0, TYPE_UINT64, 0}, \
{"usrp-tx-thread-config", CONFIG_HLP_USRP_THREAD, 0, iptr:&usrp_tx_thread, defstrval:0, TYPE_INT, 0}, \ {"usrp-tx-thread-config", CONFIG_HLP_USRP_THREAD, 0, iptr:&usrp_tx_thread, defstrval:0, TYPE_INT, 0}, \
{"ldpc-offload-enable", CONFIG_HLP_LDPC_OFFLOAD, 0, iptr:&ldpc_offload_flag, defstrval:0, TYPE_INT, 0}, \ {"ldpc-offload", CONFIG_HLP_LDPC_OFFLOAD,PARAMFLAG_BOOL, iptr:&ldpc_offload_flag, defstrval:0, TYPE_INT, 0}, \
{"uecap_file", CONFIG_HLP_UECAP_FILE, 0, strptr:&uecap_file, defstrval:"./uecap_ports1.xml",TYPE_STRING, 0}, \ {"uecap_file", CONFIG_HLP_UECAP_FILE, 0, strptr:&uecap_file, defstrval:"./uecap_ports1.xml",TYPE_STRING, 0}, \
{"s" , CONFIG_HLP_SNR, 0, dblptr:&snr_dB, defdblval:25, TYPE_DOUBLE, 0}, \ {"s" , CONFIG_HLP_SNR, 0, dblptr:&snr_dB, defdblval:25, TYPE_DOUBLE, 0}, \
} }
......
...@@ -55,19 +55,6 @@ ...@@ -55,19 +55,6 @@
#define MAX_QUEUES RTE_MAX_LCORE #define MAX_QUEUES RTE_MAX_LCORE
#define TEST_REPETITIONS 1 #define TEST_REPETITIONS 1
#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
#include <fpga_lte_fec.h>
#define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
#define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
#define VF_UL_4G_QUEUE_VALUE 4
#define VF_DL_4G_QUEUE_VALUE 4
#define UL_4G_BANDWIDTH 3
#define DL_4G_BANDWIDTH 3
#define UL_4G_LOAD_BALANCE 128
#define DL_4G_LOAD_BALANCE 128
#define FLR_4G_TIMEOUT 610
#endif
#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC
#include <rte_pmd_fpga_5gnr_fec.h> #include <rte_pmd_fpga_5gnr_fec.h>
#define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf") #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
...@@ -362,64 +349,17 @@ add_dev(uint8_t dev_id, struct rte_bbdev_info *info) ...@@ -362,64 +349,17 @@ add_dev(uint8_t dev_id, struct rte_bbdev_info *info)
unsigned int nb_queues; unsigned int nb_queues;
enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC; enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC;
/* Configure fpga lte fec with PF & VF values
* if '-i' flag is set and using fpga device
*/
#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
if ((get_init_device() == true) &&
(!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
struct fpga_lte_fec_conf conf;
unsigned int i;
printf("Configure FPGA LTE FEC Driver %s with default values\n",
info->drv.driver_name);
/* clear default configuration before initialization */
memset(&conf, 0, sizeof(struct fpga_lte_fec_conf));
/* Set PF mode :
* true if PF is used for data plane
* false for VFs
*/
conf.pf_mode_en = true;
for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
/* Number of UL queues per VF (fpga supports 8 VFs) */
conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
/* Number of DL queues per VF (fpga supports 8 VFs) */
conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
}
/* UL bandwidth. Needed for schedule algorithm */
conf.ul_bandwidth = UL_4G_BANDWIDTH;
/* DL bandwidth */
conf.dl_bandwidth = DL_4G_BANDWIDTH;
/* UL & DL load Balance Factor to 64 */
conf.ul_load_balance = UL_4G_LOAD_BALANCE;
conf.dl_load_balance = DL_4G_LOAD_BALANCE;
/**< FLR timeout value */
conf.flr_time_out = FLR_4G_TIMEOUT;
/* setup FPGA PF with configuration information */
ret = fpga_lte_fec_configure(info->dev_name, &conf);
TEST_ASSERT_SUCCESS(ret,
"Failed to configure 4G FPGA PF for bbdev %s",
info->dev_name);
}
#endif
#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC
if ((get_init_device() == true) && if ((get_init_device() == true) &&
(!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) { (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
struct fpga_5gnr_fec_conf conf; struct rte_fpga_5gnr_fec_conf conf;
unsigned int i; unsigned int i;
printf("Configure FPGA 5GNR FEC Driver %s with default values\n", printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
info->drv.driver_name); info->drv.driver_name);
/* clear default configuration before initialization */ /* clear default configuration before initialization */
memset(&conf, 0, sizeof(struct fpga_5gnr_fec_conf)); memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
/* Set PF mode : /* Set PF mode :
* true if PF is used for data plane * true if PF is used for data plane
...@@ -447,7 +387,7 @@ add_dev(uint8_t dev_id, struct rte_bbdev_info *info) ...@@ -447,7 +387,7 @@ add_dev(uint8_t dev_id, struct rte_bbdev_info *info)
conf.flr_time_out = FLR_5G_TIMEOUT; conf.flr_time_out = FLR_5G_TIMEOUT;
/* setup FPGA PF with configuration information */ /* setup FPGA PF with configuration information */
ret = fpga_5gnr_fec_configure(info->dev_name, &conf); ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf);
TEST_ASSERT_SUCCESS(ret, TEST_ASSERT_SUCCESS(ret,
"Failed to configure 5G FPGA PF for bbdev %s", "Failed to configure 5G FPGA PF for bbdev %s",
info->dev_name); info->dev_name);
...@@ -1260,7 +1200,7 @@ start_pmd_dec(struct active_device *ad, ...@@ -1260,7 +1200,7 @@ start_pmd_dec(struct active_device *ad,
t_params[0].harq_pid = harq_pid; t_params[0].harq_pid = harq_pid;
t_params[0].ulsch_id = ulsch_id; t_params[0].ulsch_id = ulsch_id;
RTE_LCORE_FOREACH_SLAVE(lcore_id) { RTE_LCORE_FOREACH_WORKER(lcore_id) {
if (used_cores >= num_lcores) if (used_cores >= num_lcores)
break; break;
t_params[used_cores].dev_id = ad->dev_id; t_params[used_cores].dev_id = ad->dev_id;
...@@ -1412,7 +1352,7 @@ int32_t nrLDPC_decod_offload(t_nrLDPC_dec_params* p_decParams, uint8_t harq_pid, ...@@ -1412,7 +1352,7 @@ int32_t nrLDPC_decod_offload(t_nrLDPC_dec_params* p_decParams, uint8_t harq_pid,
{ {
t_nrLDPCoffload_params offloadParams; t_nrLDPCoffload_params offloadParams;
t_nrLDPCoffload_params* p_offloadParams = &offloadParams; t_nrLDPCoffload_params* p_offloadParams = &offloadParams;
uint64_t start=rte_rdtsc_precise(); //uint64_t start=rte_rdtsc_precise();
/*uint64_t start_time;//=rte_rdtsc_precise(); /*uint64_t start_time;//=rte_rdtsc_precise();
uint64_t start_time1; //=rte_rdtsc_precise(); uint64_t start_time1; //=rte_rdtsc_precise();
uint64_t total_time=0, total_time1=0;*/ uint64_t total_time=0, total_time1=0;*/
...@@ -1421,36 +1361,28 @@ int32_t nrLDPC_decod_offload(t_nrLDPC_dec_params* p_decParams, uint8_t harq_pid, ...@@ -1421,36 +1361,28 @@ int32_t nrLDPC_decod_offload(t_nrLDPC_dec_params* p_decParams, uint8_t harq_pid,
/*uint64_t start_time_init; /*uint64_t start_time_init;
uint64_t total_time_init=0;*/ uint64_t total_time_init=0;*/
/* int argc_re = 6;
int argc_re=2; char *argv_re[6];
char *argv_re[2]; argv_re[0] = "~/dpdk-stable/build/app";
argv_re[0] = "/home/eurecom/hongzhi/dpdk-20.05orig/build/app/testbbdev"; argv_re[1] = "-a";
argv_re[1] = "--"; argv_re[2] = "0000:01:00.0";
*/ argv_re[3] = "--";
argv_re[4] = "-l";
int argc_re=7; argv_re[5] = "1-4";
char *argv_re[7];
argv_re[0] = "/home/eurecom/hongzhi/dpdk-20.05orig/build/app/testbbdev"; test_params.num_ops = 1;
argv_re[1] = "-l"; test_params.burst_sz = 1;
argv_re[2] = "31"; test_params.num_lcores = 1;
argv_re[3] = "-w"; test_params.num_tests = 1;
argv_re[4] = "b0:00.0"; struct active_device *ad;
argv_re[5] = "--file-prefix=b6"; ad = &active_devs[0];
argv_re[6] = "--";
int socket_id = 0;
test_params.num_ops=1; int i, f_ret;
test_params.burst_sz=1; struct rte_bbdev_info info;
test_params.num_lcores=1; enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC;
test_params.num_tests = 1;
struct active_device *ad; switch (mode) {
ad = &active_devs[0];
int socket_id=0;
int i,f_ret;
struct rte_bbdev_info info;
enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC;
switch (mode) {
case 0: case 0:
ret = rte_eal_init(argc_re, argv_re); ret = rte_eal_init(argc_re, argv_re);
if (ret<0) { if (ret<0) {
......
...@@ -75,7 +75,7 @@ int load_nrLDPClib_offload(void) { ...@@ -75,7 +75,7 @@ int load_nrLDPClib_offload(void) {
loader_shlibfunc_t shlib_decoffload_fdesc; loader_shlibfunc_t shlib_decoffload_fdesc;
shlib_decoffload_fdesc.fname = "nrLDPC_decod_offload"; shlib_decoffload_fdesc.fname = "nrLDPC_decod_offload";
int ret=load_module_shlib("ldpc_t1",&shlib_decoffload_fdesc,1,NULL); int ret=load_module_shlib("ldpc_t2",&shlib_decoffload_fdesc,1,NULL);
AssertFatal( (ret >= 0),"Error loading ldpc decoder offload"); AssertFatal( (ret >= 0),"Error loading ldpc decoder offload");
nrLDPC_decoder_offload = (nrLDPC_decoffloadfunc_t)shlib_decoffload_fdesc.fptr; nrLDPC_decoder_offload = (nrLDPC_decoffloadfunc_t)shlib_decoffload_fdesc.fptr;
......
...@@ -551,7 +551,8 @@ int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm, ...@@ -551,7 +551,8 @@ int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
printf("nr_rate_matching_ldpc_rx: Clear %d, E %d, k0 %d, Ncb %d, rvidx %d, Tbslbrm %d\n", clear, E, ind, Ncb, rvidx, Tbslbrm); printf("nr_rate_matching_ldpc_rx: Clear %d, E %d, k0 %d, Ncb %d, rvidx %d, Tbslbrm %d\n", clear, E, ind, Ncb, rvidx, Tbslbrm);
#endif #endif
if (clear==1) memset(w,0,Ncb*sizeof(int16_t)); if (clear == 1)
memset(w, 0, (68 * 384) * sizeof(int16_t));
k=0; k=0;
......
...@@ -339,7 +339,7 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB, ...@@ -339,7 +339,7 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
LOG_E(PHY,"ulsch_decoding.c: NULL harq_process pointer\n"); LOG_E(PHY,"ulsch_decoding.c: NULL harq_process pointer\n");
return 1; return 1;
} }
uint8_t dtx_det = 0; //uint8_t dtx_det = 0;
int Kr; int Kr;
int Kr_bytes; int Kr_bytes;
...@@ -362,7 +362,7 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB, ...@@ -362,7 +362,7 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_ULSCH_DECODING,1); VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_ULSCH_DECODING,1);
harq_process->TBS = pusch_pdu->pusch_data.tb_size; harq_process->TBS = pusch_pdu->pusch_data.tb_size;
dtx_det = 0; //dtx_det = 0;
uint32_t A = (harq_process->TBS) << 3; uint32_t A = (harq_process->TBS) << 3;
...@@ -461,8 +461,8 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB, ...@@ -461,8 +461,8 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
if (phy_vars_gNB->ldpc_offload_flag) { if (phy_vars_gNB->ldpc_offload_flag) {
int8_t llrProcBuf[22 * 384]; int8_t llrProcBuf[22 * 384];
// if (dtx_det==0) { // if (dtx_det==0) {
int16_t z_ol[68 * 384]; int16_t z_ol[68 * 384 + 16];
int8_t l_ol[68 * 384]; int8_t l_ol[68 * 384 + 16];
int16_t z[68 * 384 + 16] __attribute__((aligned(16))); int16_t z[68 * 384 + 16] __attribute__((aligned(16)));
int8_t l[68 * 384 + 16] __attribute__((aligned(16))); int8_t l[68 * 384 + 16] __attribute__((aligned(16)));
...@@ -490,9 +490,9 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB, ...@@ -490,9 +490,9 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
decParams.R = nr_get_R_ldpc_decoder(pusch_pdu->pusch_data.rv_index, E, decParams.BG, decParams.Z, &harq_process->llrLen, harq_process->round); decParams.R = nr_get_R_ldpc_decoder(pusch_pdu->pusch_data.rv_index, E, decParams.BG, decParams.Z, &harq_process->llrLen, harq_process->round);
if ((dtx_det == 0) && (pusch_pdu->pusch_data.rv_index == 0)) { //if ((dtx_det == 0) && (pusch_pdu->pusch_data.rv_index == 0)) {
// if (dtx_det==0){ // if (dtx_det==0){
if (mcs > 9) { if (mcs >= 0) {
memcpy((&z_ol[0]), ulsch_llr + r_offset, E * sizeof(short)); memcpy((&z_ol[0]), ulsch_llr + r_offset, E * sizeof(short));
__m128i *pv_ol128 = (__m128i *)&z_ol; __m128i *pv_ol128 = (__m128i *)&z_ol;
__m128i *pl_ol128 = (__m128i *)&l_ol; __m128i *pl_ol128 = (__m128i *)&l_ol;
...@@ -575,10 +575,10 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB, ...@@ -575,10 +575,10 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
printf("llrprocbuf [%d] = %x adr %p\n", k, llrProcBuf[k], llrProcBuf+k); printf("llrprocbuf [%d] = %x adr %p\n", k, llrProcBuf[k], llrProcBuf+k);
} }
*/ */
} else { // } else {
dtx_det = 0; // dtx_det = 0;
no_iteration_ldpc = ulsch->max_ldpc_iterations + 1; // no_iteration_ldpc = ulsch->max_ldpc_iterations + 1;
} // }
bool decodeSuccess = (no_iteration_ldpc <= ulsch->max_ldpc_iterations); bool decodeSuccess = (no_iteration_ldpc <= ulsch->max_ldpc_iterations);
if (decodeSuccess) { if (decodeSuccess) {
memcpy(harq_process->b + offset, harq_process->c[r], Kr_bytes - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0)); memcpy(harq_process->b + offset, harq_process->c[r], Kr_bytes - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
...@@ -621,7 +621,7 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB, ...@@ -621,7 +621,7 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
} }
else { else {
dtx_det = 0; //dtx_det = 0;
for (int r = 0; r < harq_process->C; r++) { for (int r = 0; r < harq_process->C; r++) {
int E = nr_get_E(G, harq_process->C, Qm, n_layers, r); int E = nr_get_E(G, harq_process->C, Qm, n_layers, r);
......
...@@ -11,7 +11,7 @@ gNBs = ...@@ -11,7 +11,7 @@ gNBs =
// Tracking area code, 0x0000 and 0xfffe are reserved values // Tracking area code, 0x0000 and 0xfffe are reserved values
tracking_area_code = 1; tracking_area_code = 1;
plmn_list = ({ mcc = 001; mnc = 01; mnc_length = 2; snssaiList = ({ sst = 1; sd = 0x1; }) }); plmn_list = ({ mcc = 001; mnc = 01; mnc_length = 2; snssaiList = ({ sst = 1; }) });
nr_cellid = 12345678L; nr_cellid = 12345678L;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment