Commit da9b8791 authored by Raymond Knopp's avatar Raymond Knopp

Merge remote-tracking branch 'origin/develop-nr' into develop-nr-less-logging

Conflicts:
	openair1/SCHED_NR/sched_nr.h
parents 01879c3e 19558fda
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
<!--
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The OpenAirInterface Software Alliance licenses this file to You under
the OAI Public License, Version 1.1 (the "License"); you may not use this file
except in compliance with the License.
You may obtain a copy of the License at
http://www.openairinterface.org/?page_id=698
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
For more information about the OpenAirInterface (OAI) Software Alliance:
contact@openairinterface.org
-->
<testCaseList>
<htmlTabRef>run-OAI-gNB-NR-UE-USRP</htmlTabRef>
<htmlTabName>run OAI gNB and OAI NR UE USRP</htmlTabName>
<htmlTabIcon>tasks</htmlTabIcon>
<TestCaseRequestedList>
090101 000001 090102 000001 090108 090109
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="090101">
<class>Initialize_OAI_eNB</class>
<desc>Initialize gNB USRP</desc>
<Initialize_OAI_eNB_args>-O ../../../ci-scripts/conf_files/gnb.band78.tm1.106PRB.usrpn300.conf</Initialize_OAI_eNB_args>
<air_interface>NR</air_interface>
</testCase>
<testCase id="000001">
<class>IdleSleep</class>
<desc>Sleep</desc>
<idle_sleep_time_in_sec>5</idle_sleep_time_in_sec>
</testCase>
<testCase id="090102">
<class>Initialize_OAI_UE</class>
<desc>Initialize NR UE USRP</desc>
<Initialize_OAI_UE_args>-C 3510000000 --numerology 1 -r 106 --phy-test --usrp-args "addr=192.168.30.2,clock_source=external,time_source=external" --threadoffset 16</Initialize_OAI_UE_args>
<air_interface>NR</air_interface>
</testCase>
<testCase id="090108">
<class>Terminate_OAI_UE</class>
<desc>Terminate NR UE</desc>
<air_interface>NR</air_interface>
</testCase>
<testCase id="090109">
<class>Terminate_eNB</class>
<desc>Terminate gNB</desc>
<air_interface>NR</air_interface>
</testCase>
</testCaseList>
<!--
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The OpenAirInterface Software Alliance licenses this file to You under
the OAI Public License, Version 1.1 (the "License"); you may not use this file
except in compliance with the License.
You may obtain a copy of the License at
http://www.openairinterface.org/?page_id=698
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
For more information about the OpenAirInterface (OAI) Software Alliance:
contact@openairinterface.org
-->
<testCaseList>
<htmlTabRef>gnb-build-tab</htmlTabRef>
<htmlTabName>Build-gNB</htmlTabName>
<htmlTabIcon>wrench</htmlTabIcon>
<TestCaseRequestedList>
010101
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="010101">
<mode>TesteNB</mode>
<class>Build_eNB</class>
<desc>Build gNB (USRP)</desc>
<Build_eNB_args>--gNB -w USRP</Build_eNB_args>
</testCase>
</testCaseList>
<!--
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The OpenAirInterface Software Alliance licenses this file to You under
the OAI Public License, Version 1.1 (the "License"); you may not use this file
except in compliance with the License.
You may obtain a copy of the License at
http://www.openairinterface.org/?page_id=698
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
For more information about the OpenAirInterface (OAI) Software Alliance:
contact@openairinterface.org
-->
<testCaseList>
<htmlTabRef>nr-ue-build-tab</htmlTabRef>
<htmlTabName>Build-NR-UE</htmlTabName>
<htmlTabIcon>wrench</htmlTabIcon>
<TestCaseRequestedList>
010102
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="010102">
<mode>TestUE</mode>
<class>Build_OAI_UE</class>
<desc>Build NR UE (USRP)</desc>
<Build_OAI_UE_args>--nrUE -w USRP</Build_OAI_UE_args>
<clean_repository>false</clean_repository>
</testCase>
</testCaseList>
...@@ -754,6 +754,7 @@ include_directories("${OPENAIR_BIN_DIR}") ...@@ -754,6 +754,7 @@ include_directories("${OPENAIR_BIN_DIR}")
# Legacy exact order # Legacy exact order
include_directories("${OPENAIR_DIR}/executables")
include_directories("${OPENAIR2_DIR}/COMMON") include_directories("${OPENAIR2_DIR}/COMMON")
include_directories("${OPENAIR2_DIR}/UTIL") include_directories("${OPENAIR2_DIR}/UTIL")
include_directories("${OPENAIR2_DIR}/UTIL/LOG") include_directories("${OPENAIR2_DIR}/UTIL/LOG")
...@@ -2113,11 +2114,18 @@ if (${XFORMS}) ...@@ -2113,11 +2114,18 @@ if (${XFORMS})
${OPENAIR1_DIR}/PHY/TOOLS/nr_phy_scope.c ${OPENAIR1_DIR}/PHY/TOOLS/nr_phy_scope.c
) )
set(XFORMS_SOURCE_SOFTMODEM set(XFORMS_SOURCE_SOFTMODEM
${OPENAIR_TARGETS}/RT/USER/stats.c ${OPENAIR_DIR}/targets/RT/USER/stats.c
) )
set(XFORMS_LIBRARIES "forms") set(XFORMS_LIBRARIES "forms")
endif (${XFORMS}) endif (${XFORMS})
include_directories ("/usr/include/X11")
add_library(gnbScope MODULE
${OPENAIR1_DIR}/PHY/TOOLS/nr_phy_scope.c
)
target_link_libraries(gnbScope forms)
set(CMAKE_MODULE_PATH "${OPENAIR_DIR}/cmake_targets/tools/MODULES" "${CMAKE_MODULE_PATH}") set(CMAKE_MODULE_PATH "${OPENAIR_DIR}/cmake_targets/tools/MODULES" "${CMAKE_MODULE_PATH}")
#include T directory even if the T is off because T macros are in the code #include T directory even if the T is off because T macros are in the code
...@@ -2161,6 +2169,24 @@ add_definitions(-DASN1_MINIMUM_VERSION=924) ...@@ -2161,6 +2169,24 @@ add_definitions(-DASN1_MINIMUM_VERSION=924)
################################# #################################
# add executables for operation # add executables for operation
################################# #################################
add_library(minimal_lib
${OPENAIR_DIR}/common/utils/backtrace.c
${OPENAIR_DIR}/common/utils/LOG/log.c
${OPENAIR_DIR}/common/config/config_userapi.c
${OPENAIR_DIR}/common/config/config_load_configmodule.c
${OPENAIR_DIR}/common/config/config_cmdline.c
${OPENAIR_DIR}/common/utils/minimal_stub.c
)
target_link_libraries(minimal_lib pthread dl)
add_executable(replay_node
${OPENAIR_TARGETS}/ARCH/rfsimulator/stored_node.c
)
target_link_libraries (replay_node minimal_lib)
add_executable(measurement_display
${OPENAIR_DIR}/common/utils/threadPool/measurement_display.c)
target_link_libraries (measurement_display minimal_lib)
# lte-softmodem is both eNB and UE implementation # lte-softmodem is both eNB and UE implementation
################################################### ###################################################
...@@ -2323,19 +2349,16 @@ add_executable(nr-softmodem ...@@ -2323,19 +2349,16 @@ add_executable(nr-softmodem
${nr_rrc_h} ${nr_rrc_h}
${s1ap_h} ${s1ap_h}
# ${OPENAIR_BIN_DIR}/messages_xml.h # ${OPENAIR_BIN_DIR}/messages_xml.h
${OPENAIR_TARGETS}/RT/USER/rt_wrapper.c ${OPENAIR_DIR}/executables/nr-gnb.c
${OPENAIR_TARGETS}/RT/USER/nr-gnb.c ${OPENAIR_DIR}/executables/nr-ru.c
${OPENAIR_TARGETS}/RT/USER/nr-ru.c ${OPENAIR_DIR}/executables/nr-softmodem.c
${OPENAIR_TARGETS}/RT/USER/nr-softmodem.c
${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c ${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c
${OPENAIR_TARGETS}/COMMON/create_nr_tasks.c
${OPENAIR_TARGETS}/ARCH/COMMON/common_lib.c ${OPENAIR_TARGETS}/ARCH/COMMON/common_lib.c
${OPENAIR1_DIR}/SIMULATION/ETH_TRANSPORT/netlink_init.c ${OPENAIR1_DIR}/SIMULATION/ETH_TRANSPORT/netlink_init.c
${OPENAIR_DIR}/common/utils/utils.c ${OPENAIR_DIR}/common/utils/utils.c
${OPENAIR_DIR}/common/utils/system.c ${OPENAIR_DIR}/common/utils/system.c
${GTPU_need_ITTI} ${GTPU_need_ITTI}
${XFORMS_SOURCE} ${XFORMS_SOURCE_NR}
${XFORMS_SOURCE_SOFTMODEM}
${T_SOURCE} ${T_SOURCE}
${CONFIG_SOURCES} ${CONFIG_SOURCES}
${SHLIB_LOADER_SOURCES} ${SHLIB_LOADER_SOURCES}
...@@ -2361,13 +2384,10 @@ add_executable(nr-softmodem-nos1 ...@@ -2361,13 +2384,10 @@ add_executable(nr-softmodem-nos1
${rrc_h} ${rrc_h}
${s1ap_h} ${s1ap_h}
# ${OPENAIR_BIN_DIR}/messages_xml.h # ${OPENAIR_BIN_DIR}/messages_xml.h
${OPENAIR_TARGETS}/RT/USER/rt_wrapper.c ${OPENAIR_DIR}/executables/nr-gnb.c
${OPENAIR_TARGETS}/RT/USER/nr-gnb.c ${OPENAIR_DIR}/executables/nr-ru.c
${OPENAIR_TARGETS}/RT/USER/nr-ru.c ${OPENAIR_DIR}/executables/nr-softmodem.c
${OPENAIR_TARGETS}/RT/USER/nr-softmodem.c
${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c ${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c
${OPENAIR_TARGETS}/COMMON/create_tasks.c
${OPENAIR_TARGETS}/COMMON/create_nr_tasks.c
${OPENAIR_TARGETS}/ARCH/COMMON/common_lib.c ${OPENAIR_TARGETS}/ARCH/COMMON/common_lib.c
${OPENAIR2_DIR}/RRC/NAS/nas_config.c ${OPENAIR2_DIR}/RRC/NAS/nas_config.c
${OPENAIR2_DIR}/RRC/NAS/rb_config.c ${OPENAIR2_DIR}/RRC/NAS/rb_config.c
...@@ -2375,8 +2395,7 @@ add_executable(nr-softmodem-nos1 ...@@ -2375,8 +2395,7 @@ add_executable(nr-softmodem-nos1
${OPENAIR_DIR}/common/utils/utils.c ${OPENAIR_DIR}/common/utils/utils.c
${OPENAIR_DIR}/common/utils/system.c ${OPENAIR_DIR}/common/utils/system.c
${GTPU_need_ITTI} ${GTPU_need_ITTI}
${XFORMS_SOURCE} ${XFORMS_SOURCE_NR}
${XFORMS_SOURCE_SOFTMODEM}
${T_SOURCE} ${T_SOURCE}
${CONFIG_SOURCES} ${CONFIG_SOURCES}
${SHLIB_LOADER_SOURCES} ${SHLIB_LOADER_SOURCES}
...@@ -2412,7 +2431,6 @@ add_executable(nr-uesoftmodem ...@@ -2412,7 +2431,6 @@ add_executable(nr-uesoftmodem
${OPENAIR_DIR}/common/utils/utils.c ${OPENAIR_DIR}/common/utils/utils.c
${OPENAIR_DIR}/common/utils/system.c ${OPENAIR_DIR}/common/utils/system.c
${XFORMS_SOURCE_NR} ${XFORMS_SOURCE_NR}
${XFORMS_SOURCE_SOFTMODEM}
${T_SOURCE} ${T_SOURCE}
${CONFIG_SOURCES} ${CONFIG_SOURCES}
${SHLIB_LOADER_SOURCES} ${SHLIB_LOADER_SOURCES}
...@@ -2436,17 +2454,14 @@ add_executable(nr-uesoftmodem-nos1 ...@@ -2436,17 +2454,14 @@ add_executable(nr-uesoftmodem-nos1
${rrc_h} ${rrc_h}
${s1ap_h} ${s1ap_h}
# ${OPENAIR_BIN_DIR}/messages_xml.h # ${OPENAIR_BIN_DIR}/messages_xml.h
${OPENAIR_TARGETS}/RT/USER/rt_wrapper.c
${OPENAIR_DIR}/executables/nr-ue.c ${OPENAIR_DIR}/executables/nr-ue.c
${OPENAIR_DIR}/executables/nr-uesoftmodem.c ${OPENAIR_DIR}/executables/nr-uesoftmodem.c
${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c ${OPENAIR1_DIR}/SIMULATION/TOOLS/taus.c
#${OPENAIR_TARGETS}/COMMON/create_tasks_ue.c
${OPENAIR_TARGETS}/ARCH/COMMON/common_lib.c ${OPENAIR_TARGETS}/ARCH/COMMON/common_lib.c
${OPENAIR1_DIR}/SIMULATION/ETH_TRANSPORT/netlink_init.c ${OPENAIR1_DIR}/SIMULATION/ETH_TRANSPORT/netlink_init.c
${OPENAIR_DIR}/common/utils/utils.c ${OPENAIR_DIR}/common/utils/utils.c
${OPENAIR_DIR}/common/utils/system.c ${OPENAIR_DIR}/common/utils/system.c
${XFORMS_SOURCE_NR} ${XFORMS_SOURCE_NR}
${XFORMS_SOURCE_SOFTMODEM}
${T_SOURCE} ${T_SOURCE}
${CONFIG_SOURCES} ${CONFIG_SOURCES}
${SHLIB_LOADER_SOURCES} ${SHLIB_LOADER_SOURCES}
...@@ -2498,6 +2513,7 @@ target_link_libraries (nr-uesoftmodem-nos1 ${T_LIB}) ...@@ -2498,6 +2513,7 @@ target_link_libraries (nr-uesoftmodem-nos1 ${T_LIB})
add_executable(dlsim_tm4 add_executable(dlsim_tm4
${OPENAIR1_DIR}/SIMULATION/LTE_PHY/dlsim_tm4.c ${OPENAIR1_DIR}/SIMULATION/LTE_PHY/dlsim_tm4.c
${OPENAIR1_DIR}/PHY/TOOLS/lte_phy_scope_tm4.c ${OPENAIR1_DIR}/PHY/TOOLS/lte_phy_scope_tm4.c
${OPENAIR_DIR}/common/utils/system.c
${T_SOURCE} ${T_SOURCE}
) )
target_link_libraries (dlsim_tm4 target_link_libraries (dlsim_tm4
...@@ -2524,18 +2540,21 @@ target_link_libraries(ldpctest SIMU PHY PHY_NR m ${ATLAS_LIBRARIES}) ...@@ -2524,18 +2540,21 @@ target_link_libraries(ldpctest SIMU PHY PHY_NR m ${ATLAS_LIBRARIES})
add_executable(nr_dlschsim add_executable(nr_dlschsim
${OPENAIR1_DIR}/SIMULATION/NR_PHY/dlschsim.c ${OPENAIR1_DIR}/SIMULATION/NR_PHY/dlschsim.c
${OPENAIR_DIR}/common/utils/backtrace.c ${OPENAIR_DIR}/common/utils/backtrace.c
${OPENAIR_DIR}/common/utils/system.c
${T_SOURCE}) ${T_SOURCE})
target_link_libraries(nr_dlschsim -Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR PHY_NR_UE SCHED_NR_LIB CONFIG_LIB -Wl,--end-group m pthread ${ATLAS_LIBRARIES} ${T_LIB} dl) target_link_libraries(nr_dlschsim -Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR PHY_NR_UE SCHED_NR_LIB CONFIG_LIB -Wl,--end-group m pthread ${ATLAS_LIBRARIES} ${T_LIB} dl)
add_executable(nr_pbchsim add_executable(nr_pbchsim
${OPENAIR1_DIR}/SIMULATION/NR_PHY/pbchsim.c ${OPENAIR1_DIR}/SIMULATION/NR_PHY/pbchsim.c
${OPENAIR_DIR}/common/utils/backtrace.c ${OPENAIR_DIR}/common/utils/backtrace.c
${OPENAIR_DIR}/common/utils/system.c
${T_SOURCE}) ${T_SOURCE})
target_link_libraries(nr_pbchsim -Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR PHY_NR_UE SCHED_NR_LIB CONFIG_LIB -Wl,--end-group m pthread ${ATLAS_LIBRARIES} ${T_LIB} dl) target_link_libraries(nr_pbchsim -Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR PHY_NR_UE SCHED_NR_LIB CONFIG_LIB -Wl,--end-group m pthread ${ATLAS_LIBRARIES} ${T_LIB} dl)
add_executable(nr_dlsim add_executable(nr_dlsim
${OPENAIR1_DIR}/SIMULATION/NR_PHY/dlsim.c ${OPENAIR1_DIR}/SIMULATION/NR_PHY/dlsim.c
${OPENAIR_DIR}/common/utils/backtrace.c ${OPENAIR_DIR}/common/utils/backtrace.c
${OPENAIR_DIR}/common/utils/system.c
${T_SOURCE}) ${T_SOURCE})
target_link_libraries(nr_dlsim -Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR PHY_NR_UE SCHED_NR_LIB SCHED_NR_UE_LIB MAC_NR MAC_UE_NR MAC_NR_COMMON RRC_LIB NR_RRC_LIB CONFIG_LIB L2_NR -Wl,--end-group m pthread ${ATLAS_LIBRARIES} ${T_LIB} dl) target_link_libraries(nr_dlsim -Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR PHY_NR_UE SCHED_NR_LIB SCHED_NR_UE_LIB MAC_NR MAC_UE_NR MAC_NR_COMMON RRC_LIB NR_RRC_LIB CONFIG_LIB L2_NR -Wl,--end-group m pthread ${ATLAS_LIBRARIES} ${T_LIB} dl)
...@@ -2551,6 +2570,7 @@ foreach(myExe dlsim dlsim_tm7 ulsim pbchsim scansim mbmssim pdcchsim pucchsim pr ...@@ -2551,6 +2570,7 @@ foreach(myExe dlsim dlsim_tm7 ulsim pbchsim scansim mbmssim pdcchsim pucchsim pr
add_executable(${myExe} add_executable(${myExe}
${OPENAIR1_DIR}/SIMULATION/LTE_PHY/${myExe}.c ${OPENAIR1_DIR}/SIMULATION/LTE_PHY/${myExe}.c
${OPENAIR_DIR}/common/utils/backtrace.c ${OPENAIR_DIR}/common/utils/backtrace.c
${OPENAIR_DIR}/common/utils/system.c
${XFORMS_SOURCE} ${XFORMS_SOURCE}
${T_SOURCE} ${T_SOURCE}
${CONFIG_SOURCES} ${CONFIG_SOURCES}
......
...@@ -420,11 +420,6 @@ function main() { ...@@ -420,11 +420,6 @@ function main() {
CMAKE_CMD="$CMAKE_CMD .." CMAKE_CMD="$CMAKE_CMD .."
echo_info "CMAKE_CMD=$CMAKE_CMD" echo_info "CMAKE_CMD=$CMAKE_CMD"
if [ "$eNB" = "1" -o "$gNB" = "1" ] && [ "$UE" = "1" -o "$nrUE" = "1" ]; then
echo_error "Cannot build UE/nrUE and eNB/gNB on one build_oai execution"
echo_error "use 2 build_oai invocations"
exit
fi
######################################################### #########################################################
# check validity of HW and TP parameters for eNB / gNB # check validity of HW and TP parameters for eNB / gNB
......
int T_stdout;
void exit_function(const char *file, const char *function, const int line, const char *s) {
}
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
#include <intertask_interface.h> #include <intertask_interface.h>
#include <common/utils/system.h>
typedef struct timer_elm_s { typedef struct timer_elm_s {
timer_type_t type; ///< Timer type timer_type_t type; ///< Timer type
...@@ -280,22 +281,8 @@ extern "C" { ...@@ -280,22 +281,8 @@ extern "C" {
int itti_create_task(task_id_t task_id, void *(*start_routine)(void *), void *args_p) { int itti_create_task(task_id_t task_id, void *(*start_routine)(void *), void *args_p) {
task_list_t *t=&tasks[task_id]; task_list_t *t=&tasks[task_id];
AssertFatal ( pthread_create (&t->thread, NULL, start_routine, args_p) ==0, threadCreate (&t->thread, start_routine, args_p, (char*)itti_get_task_name(task_id),-1,OAI_PRIORITY_RT);
"Thread creation for task %d failed!\n", task_id);
pthread_setname_np( t->thread, itti_get_task_name(task_id) );
LOG_I(TMR,"Created Posix thread %s\n", itti_get_task_name(task_id) ); LOG_I(TMR,"Created Posix thread %s\n", itti_get_task_name(task_id) );
#if 1 // BMC test RT prio
{
int policy;
struct sched_param sparam;
memset(&sparam, 0, sizeof(sparam));
sparam.sched_priority = sched_get_priority_max(SCHED_FIFO)-10;
policy = SCHED_FIFO ;
if (pthread_setschedparam(t->thread, policy, &sparam) != 0) {
LOG_E(TMR,"task %s : Failed to set pthread priority\n", itti_get_task_name(task_id) );
}
}
#endif
return 0; return 0;
} }
......
#ifndef __SIMPLE_EXE_H__
#define __SIMPLE_EXE_H__
#ifndef __USE_GNU
#define __USE_GNU
#endif
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <stdio.h>
#include <pthread.h>
#include <sched.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <string.h>
#include <unistd.h>
#include <sys/syscall.h>
#include <sys/time.h>
#include <stdint.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <stdbool.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <common/utils/assertions.h>
#include <common/utils/LOG/log.h>
#include "common_lib.h"
#ifdef T
#undef T
#define T(...)
#endif
#endif
...@@ -28,13 +28,23 @@ ...@@ -28,13 +28,23 @@
* separate process solves this problem. * separate process solves this problem.
*/ */
#define _GNU_SOURCE
#include "system.h" #include "system.h"
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include <pthread.h> #include <pthread.h>
#include <string.h> #include <string.h>
#include <stdint.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>
#include <pthread.h>
#include <common/utils/assertions.h>
#include <common/utils/LOG/log.h>
#define MAX_COMMAND 4096 #define MAX_COMMAND 4096
static int command_pipe_read; static int command_pipe_read;
...@@ -50,37 +60,37 @@ static int module_initialized = 0; ...@@ -50,37 +60,37 @@ static int module_initialized = 0;
/* util functions */ /* util functions */
/********************************************************************/ /********************************************************************/
static void lock_system(void) static void lock_system(void) {
{
if (pthread_mutex_lock(&lock) != 0) { if (pthread_mutex_lock(&lock) != 0) {
printf("pthread_mutex_lock fails\n"); printf("pthread_mutex_lock fails\n");
abort(); abort();
} }
} }
static void unlock_system(void) static void unlock_system(void) {
{
if (pthread_mutex_unlock(&lock) != 0) { if (pthread_mutex_unlock(&lock) != 0) {
printf("pthread_mutex_unlock fails\n"); printf("pthread_mutex_unlock fails\n");
abort(); abort();
} }
} }
static void write_pipe(int p, char *b, int size) static void write_pipe(int p, char *b, int size) {
{
while (size) { while (size) {
int ret = write(p, b, size); int ret = write(p, b, size);
if (ret <= 0) exit(0); if (ret <= 0) exit(0);
b += ret; b += ret;
size -= ret; size -= ret;
} }
} }
static void read_pipe(int p, char *b, int size) static void read_pipe(int p, char *b, int size) {
{
while (size) { while (size) {
int ret = read(p, b, size); int ret = read(p, b, size);
if (ret <= 0) exit(0); if (ret <= 0) exit(0);
b += ret; b += ret;
size -= ret; size -= ret;
} }
...@@ -95,14 +105,13 @@ static void read_pipe(int p, char *b, int size) ...@@ -95,14 +105,13 @@ static void read_pipe(int p, char *b, int size)
* when the main process exits, because then a "read" on the pipe * when the main process exits, because then a "read" on the pipe
* will return 0, in which case "read_pipe" exits. * will return 0, in which case "read_pipe" exits.
*/ */
static void background_system_process(void) static void background_system_process(void) {
{
int len; int len;
int ret; int ret;
char command[MAX_COMMAND+1]; char command[MAX_COMMAND+1];
while (1) { while (1) {
read_pipe(command_pipe_read, (char*)&len, sizeof(int)); read_pipe(command_pipe_read, (char *)&len, sizeof(int));
read_pipe(command_pipe_read, command, len); read_pipe(command_pipe_read, command, len);
ret = system(command); ret = system(command);
write_pipe(result_pipe_write, (char *)&ret, sizeof(int)); write_pipe(result_pipe_write, (char *)&ret, sizeof(int));
...@@ -114,8 +123,7 @@ static void background_system_process(void) ...@@ -114,8 +123,7 @@ static void background_system_process(void)
/* return -1 on error, 0 on success */ /* return -1 on error, 0 on success */
/********************************************************************/ /********************************************************************/
int background_system(char *command) int background_system(char *command) {
{
int res; int res;
int len; int len;
...@@ -125,18 +133,22 @@ int background_system(char *command) ...@@ -125,18 +133,22 @@ int background_system(char *command)
} }
len = strlen(command)+1; len = strlen(command)+1;
if (len > MAX_COMMAND) { if (len > MAX_COMMAND) {
printf("FATAL: command too long. Increase MAX_COMMAND (%d).\n", MAX_COMMAND); printf("FATAL: command too long. Increase MAX_COMMAND (%d).\n", MAX_COMMAND);
printf("command was: '%s'\n", command); printf("command was: '%s'\n", command);
abort(); abort();
} }
/* only one command can run at a time, so let's lock/unlock */ /* only one command can run at a time, so let's lock/unlock */
lock_system(); lock_system();
write_pipe(command_pipe_write, (char*)&len, sizeof(int)); write_pipe(command_pipe_write, (char *)&len, sizeof(int));
write_pipe(command_pipe_write, command, len); write_pipe(command_pipe_write, command, len);
read_pipe(result_pipe_read, (char*)&res, sizeof(int)); read_pipe(result_pipe_read, (char *)&res, sizeof(int));
unlock_system(); unlock_system();
if (res == -1 || !WIFEXITED(res) || WEXITSTATUS(res) != 0) return -1; if (res == -1 || !WIFEXITED(res) || WEXITSTATUS(res) != 0) return -1;
return 0; return 0;
} }
...@@ -146,17 +158,16 @@ int background_system(char *command) ...@@ -146,17 +158,16 @@ int background_system(char *command)
/* to be called very early by the main processing */ /* to be called very early by the main processing */
/********************************************************************/ /********************************************************************/
void start_background_system(void) void start_background_system(void) {
{
int p[2]; int p[2];
pid_t son; pid_t son;
module_initialized = 1; module_initialized = 1;
if (pipe(p) == -1) { if (pipe(p) == -1) {
perror("pipe"); perror("pipe");
exit(1); exit(1);
} }
command_pipe_read = p[0]; command_pipe_read = p[0];
command_pipe_write = p[1]; command_pipe_write = p[1];
...@@ -164,10 +175,11 @@ void start_background_system(void) ...@@ -164,10 +175,11 @@ void start_background_system(void)
perror("pipe"); perror("pipe");
exit(1); exit(1);
} }
result_pipe_read = p[0]; result_pipe_read = p[0];
result_pipe_write = p[1]; result_pipe_write = p[1];
son = fork(); son = fork();
if (son == -1) { if (son == -1) {
perror("fork"); perror("fork");
exit(1); exit(1);
...@@ -181,6 +193,56 @@ void start_background_system(void) ...@@ -181,6 +193,56 @@ void start_background_system(void)
close(result_pipe_read); close(result_pipe_read);
close(command_pipe_write); close(command_pipe_write);
background_system_process(); background_system_process();
} }
void threadCreate(pthread_t* t, void * (*func)(void*), void * param, char* name, int affinity, int priority){
pthread_attr_t attr;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED);
pthread_attr_setschedpolicy(&attr, SCHED_FIFO);
struct sched_param sparam={0};
sparam.sched_priority = priority;
pthread_attr_setschedparam(&attr, &sparam);
pthread_create(t, &attr, func, param);
pthread_setname_np(*t, name);
if (affinity != -1 ) {
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(affinity, &cpuset);
AssertFatal( pthread_setaffinity_np(*t, sizeof(cpu_set_t), &cpuset) == 0, "Error setting processor affinity");
}
pthread_attr_destroy(&attr);
}
// Block CPU C-states deep sleep
void configure_linux(void) {
int ret;
static int latency_target_fd=-1;
uint32_t latency_target_value=10; // in microseconds
if (latency_target_fd == -1) {
if ( (latency_target_fd = open("/dev/cpu_dma_latency", O_RDWR)) != -1 ) {
ret = write(latency_target_fd, &latency_target_value, sizeof(latency_target_value));
if (ret == 0) {
printf("# error setting cpu_dma_latency to %d!: %s\n", latency_target_value, strerror(errno));
close(latency_target_fd);
latency_target_fd=-1;
return;
}
}
}
if (latency_target_fd != -1)
LOG_I(HW,"# /dev/cpu_dma_latency set to %dus\n", latency_target_value);
else
LOG_E(HW,"Can't set /dev/cpu_dma_latency to %dus\n", latency_target_value);
// Set CPU frequency to it's maximum
if ( 0 != system("for d in /sys/devices/system/cpu/cpu[0-9]*; do cat $d/cpufreq/cpuinfo_max_freq > $d/cpufreq/scaling_min_freq; done"))
LOG_W(HW,"Can't set cpu frequency\n");
}
...@@ -21,6 +21,12 @@ ...@@ -21,6 +21,12 @@
#ifndef _SYSTEM_H_OAI_ #ifndef _SYSTEM_H_OAI_
#define _SYSTEM_H_OAI_ #define _SYSTEM_H_OAI_
#include <stdint.h>
#include <pthread.h>
#ifdef __cplusplus
extern "C" {
#endif
/**************************************************** /****************************************************
* send a command to the background process * send a command to the background process
...@@ -36,4 +42,23 @@ int background_system(char *command); ...@@ -36,4 +42,23 @@ int background_system(char *command);
void start_background_system(void); void start_background_system(void);
void set_latency_target(void);
void configure_linux(void);
void threadCreate(pthread_t* t, void * (*func)(void*), void * param, char* name, int affinity, int priority);
#define OAI_PRIORITY_RT_LOW sched_get_priority_min(SCHED_FIFO)
#define OAI_PRIORITY_RT sched_get_priority_max(SCHED_FIFO)-10
#define OAI_PRIORITY_RT_MAX sched_get_priority_max(SCHED_FIFO)
void thread_top_init(char *thread_name,
int affinity,
uint64_t runtime,
uint64_t deadline,
uint64_t period);
#ifdef __cplusplus
}
#endif
#endif /* _SYSTEM_H_OAI_ */ #endif /* _SYSTEM_H_OAI_ */
all: measurement_display thread-pool-test all: measurement_display thread-pool-test
measurement_display: measurement_display.c thread-pool.h measurement_display: measurement_display.c thread-pool.h
gcc measurement_display.c -I /data/openairinterface5g.nr/common/utils/ -I. /data/openairinterface5g.nr/common/utils/backtrace.c -lpthread -D TEST_THREAD_POOL -I../LOG -I../../utils/T -o measurement_display gcc measurement_display.c -I ${OPENAIR_DIR}/ -I ${OPENAIR_DIR}/common/utils/ -I. ${OPENAIR_DIR}/common/utils/backtrace.c -lpthread -D TEST_THREAD_POOL -I../LOG -I../../utils/T -o measurement_display
thread-pool-test: thread-pool.c thread-pool.h thread-pool-test: thread-pool.c thread-pool.h
gcc -g thread-pool.c -I /data/openairinterface5g.nr/common/utils/ -I. /data/openairinterface5g.nr/common/utils/backtrace.c -lpthread -D TEST_THREAD_POOL -I../LOG -I../../utils/T -o thread-pool-test gcc -g thread-pool.c -I ${OPENAIR_DIR}/ -I ${OPENAIR_DIR}/common/utils/ -I. ${OPENAIR_DIR}/common/utils/backtrace.c -I ${OPENAIR_DIR}/openair2/COMMON ${OPENAIR_DIR}/common/utils/LOG/log.c ${OPENAIR_DIR}/common/config/config_userapi.c ${OPENAIR_DIR}/common/config/config_load_configmodule.c ${OPENAIR_DIR}/common/config/config_cmdline.c -lpthread -ldl -D TEST_THREAD_POOL -I../LOG -I../../utils/T -o thread-pool-test
...@@ -2,22 +2,8 @@ ...@@ -2,22 +2,8 @@
Author: Laurent THOMAS, Open Cells Author: Laurent THOMAS, Open Cells
copyleft: OpenAirInterface Software Alliance and it's licence copyleft: OpenAirInterface Software Alliance and it's licence
*/ */
#include <common/utils/simple_executable.h>
#define __USE_GNU
#define _GNU_SOURCE
#include <stdio.h>
#include <pthread.h>
#include <sched.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <string.h>
#include <unistd.h>
#include <sys/syscall.h>
#include <sys/time.h>
#include <stdint.h>
#include <sys/stat.h>
#include <fcntl.h>
#include "thread-pool.h" #include "thread-pool.h"
#define SEP "\t" #define SEP "\t"
...@@ -26,7 +12,7 @@ uint64_t cpuCyclesMicroSec; ...@@ -26,7 +12,7 @@ uint64_t cpuCyclesMicroSec;
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
if(argc != 2) { if(argc != 2) {
printf("Need one paramter: the trace Linux pipe (fifo)"); printf("Need one parameter: the trace Linux pipe (fifo)");
exit(1); exit(1);
} }
......
...@@ -49,23 +49,6 @@ void *one_thread(void *arg) { ...@@ -49,23 +49,6 @@ void *one_thread(void *arg) {
struct one_thread *myThread=(struct one_thread *) arg; struct one_thread *myThread=(struct one_thread *) arg;
struct thread_pool *tp=myThread->pool; struct thread_pool *tp=myThread->pool;
// configure the thread core assignment
// TBD: reserve the core for us exclusively
if ( myThread->coreID >= 0 && myThread->coreID < get_nprocs_conf()) {
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(myThread->coreID, &cpuset);
pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
}
//Configure the thread scheduler policy for Linux
struct sched_param sparam= {0};
sparam.sched_priority = sched_get_priority_max(SCHED_RR);
pthread_setschedparam(pthread_self(), SCHED_RR, &sparam);
// set the thread name for debugging
sprintf(myThread->name,"Tpool_%d",myThread->coreID);
pthread_setname_np(pthread_self(), myThread->name );
// Infinite loop to process requests // Infinite loop to process requests
do { do {
notifiedFIFO_elt_t *elt=pullNotifiedFifoRemember(&tp->incomingFifo, myThread); notifiedFIFO_elt_t *elt=pullNotifiedFifoRemember(&tp->incomingFifo, myThread);
...@@ -106,10 +89,6 @@ void initTpool(char *params,tpool_t *pool, bool performanceMeas) { ...@@ -106,10 +89,6 @@ void initTpool(char *params,tpool_t *pool, bool performanceMeas) {
} else } else
pool->traceFd=-1; pool->traceFd=-1;
//Configure the thread scheduler policy for Linux
struct sched_param sparam= {0};
sparam.sched_priority = sched_get_priority_max(SCHED_RR)-1;
pthread_setschedparam(pthread_self(), SCHED_RR, &sparam);
pool->activated=true; pool->activated=true;
initNotifiedFIFO(&pool->incomingFifo); initNotifiedFIFO(&pool->incomingFifo);
char *saveptr, * curptr; char *saveptr, * curptr;
...@@ -136,7 +115,11 @@ void initTpool(char *params,tpool_t *pool, bool performanceMeas) { ...@@ -136,7 +115,11 @@ void initTpool(char *params,tpool_t *pool, bool performanceMeas) {
pool->allthreads->coreID=atoi(curptr); pool->allthreads->coreID=atoi(curptr);
pool->allthreads->id=pool->nbThreads; pool->allthreads->id=pool->nbThreads;
pool->allthreads->pool=pool; pool->allthreads->pool=pool;
pthread_create(&pool->allthreads->threadID, NULL, one_thread, (void *)pool->allthreads); //Configure the thread scheduler policy for Linux
// set the thread name for debugging
sprintf(pool->allthreads->name,"Tpool_%d",pool->allthreads->coreID);
threadCreate(&pool->allthreads->threadID, one_thread, (void *)pool->allthreads,
pool->allthreads->name, pool->allthreads->coreID, OAI_PRIORITY_RT);
pool->nbThreads++; pool->nbThreads++;
} }
...@@ -151,6 +134,9 @@ void initTpool(char *params,tpool_t *pool, bool performanceMeas) { ...@@ -151,6 +134,9 @@ void initTpool(char *params,tpool_t *pool, bool performanceMeas) {
#ifdef TEST_THREAD_POOL #ifdef TEST_THREAD_POOL
void exit_function(const char *file, const char *function, const int line, const char *s) {
}
struct testData { struct testData {
int id; int id;
char txt[30]; char txt[30];
......
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
#include <sys/syscall.h> #include <sys/syscall.h>
#include <assertions.h> #include <assertions.h>
#include <LOG/log.h> #include <LOG/log.h>
#include <common/utils/system.h>
#ifdef DEBUG #ifdef DEBUG
#define THREADINIT PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP #define THREADINIT PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP
...@@ -77,16 +78,18 @@ static inline void delNotifiedFIFO_elt(notifiedFIFO_elt_t *elt) { ...@@ -77,16 +78,18 @@ static inline void delNotifiedFIFO_elt(notifiedFIFO_elt_t *elt) {
//LOG_W(UTIL,"delNotifiedFIFO on something not allocated by newNotifiedFIFO\n"); //LOG_W(UTIL,"delNotifiedFIFO on something not allocated by newNotifiedFIFO\n");
} }
static inline void initNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf) {
nf->inF=NULL;
nf->outF=NULL;
}
static inline void initNotifiedFIFO(notifiedFIFO_t *nf) { static inline void initNotifiedFIFO(notifiedFIFO_t *nf) {
mutexinit(nf->lockF); mutexinit(nf->lockF);
condinit (nf->notifF); condinit (nf->notifF);
nf->inF=NULL; initNotifiedFIFO_nothreadSafe(nf);
nf->outF=NULL;
// No delete function: the creator has only to free the memory // No delete function: the creator has only to free the memory
} }
static inline void pushNotifiedFIFO(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg) { static inline void pushNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg) {
mutexlock(nf->lockF);
msg->next=NULL; msg->next=NULL;
if (nf->outF == NULL) if (nf->outF == NULL)
...@@ -96,22 +99,39 @@ static inline void pushNotifiedFIFO(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg) ...@@ -96,22 +99,39 @@ static inline void pushNotifiedFIFO(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg)
nf->inF->next = msg; nf->inF->next = msg;
nf->inF = msg; nf->inF = msg;
condbroadcast(nf->notifF);
mutexunlock(nf->lockF);
} }
static inline notifiedFIFO_elt_t *pullNotifiedFIFO(notifiedFIFO_t *nf) { static inline void pushNotifiedFIFO(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg) {
mutexlock(nf->lockF); mutexlock(nf->lockF);
pushNotifiedFIFO_nothreadSafe(nf,msg);
condbroadcast(nf->notifF);
mutexunlock(nf->lockF);
}
while(nf->outF == NULL) static inline notifiedFIFO_elt_t *pullNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf) {
condwait(nf->notifF, nf->lockF); if (nf->outF == NULL)
return NULL;
notifiedFIFO_elt_t *ret=nf->outF; notifiedFIFO_elt_t *ret=nf->outF;
if (nf->outF==nf->outF->next)
LOG_E(TMR,"Circular list in thread pool: push several times the same buffer is forbidden\n");
nf->outF=nf->outF->next; nf->outF=nf->outF->next;
if (nf->outF==NULL) if (nf->outF==NULL)
nf->inF=NULL; nf->inF=NULL;
return ret;
}
static inline notifiedFIFO_elt_t *pullNotifiedFIFO(notifiedFIFO_t *nf) {
mutexlock(nf->lockF);
notifiedFIFO_elt_t *ret;
while((ret=pullNotifiedFIFO_nothreadSafe(nf)) == NULL)
condwait(nf->notifF, nf->lockF);
mutexunlock(nf->lockF); mutexunlock(nf->lockF);
return ret; return ret;
} }
...@@ -122,18 +142,7 @@ static inline notifiedFIFO_elt_t *pollNotifiedFIFO(notifiedFIFO_t *nf) { ...@@ -122,18 +142,7 @@ static inline notifiedFIFO_elt_t *pollNotifiedFIFO(notifiedFIFO_t *nf) {
if (tmp != 0 ) if (tmp != 0 )
return NULL; return NULL;
notifiedFIFO_elt_t *ret=nf->outF; notifiedFIFO_elt_t *ret=pullNotifiedFIFO_nothreadSafe(nf);
if (ret!=NULL) {
if (nf->outF==nf->outF->next)
LOG_E(TMR,"Circular list in thread pool: push several times the same buffer is forbidden\n");
nf->outF=nf->outF->next;
}
if (nf->outF==NULL)
nf->inF=NULL;
mutexunlock(nf->lockF); mutexunlock(nf->lockF);
return ret; return ret;
} }
...@@ -195,7 +204,7 @@ static inline notifiedFIFO_elt_t *pullTpool(notifiedFIFO_t *responseFifo, tpool_ ...@@ -195,7 +204,7 @@ static inline notifiedFIFO_elt_t *pullTpool(notifiedFIFO_t *responseFifo, tpool_
if (t->measurePerf) if (t->measurePerf)
msg->returnTime=rdtsc(); msg->returnTime=rdtsc();
if (t->traceFd) if (t->traceFd >= 0)
if(write(t->traceFd, msg, sizeof(*msg))); if(write(t->traceFd, msg, sizeof(*msg)));
return msg; return msg;
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
...@@ -236,7 +236,6 @@ extern double cpuf; ...@@ -236,7 +236,6 @@ extern double cpuf;
extern int setup_ue_buffers(PHY_VARS_NR_UE **phy_vars_ue, openair0_config_t *openair0_cfg); extern int setup_ue_buffers(PHY_VARS_NR_UE **phy_vars_ue, openair0_config_t *openair0_cfg);
extern void fill_ue_band_info(void); extern void fill_ue_band_info(void);
extern void init_UE(int); extern void init_UE(int);
extern void init_thread(int sched_runtime, int sched_deadline, int sched_fifo, cpu_set_t *cpuset, char *name);
extern void reset_opp_meas(void); extern void reset_opp_meas(void);
extern void print_opp_meas(void); extern void print_opp_meas(void);
void *UE_thread(void *arg); void *UE_thread(void *arg);
......
This diff is collapsed.
This diff is collapsed.
#ifndef _THREADS_T_H_
#define _THREADS_T_H_
typedef struct threads_s {
int main;
int sync;
int one;
int two;
int three;
int slot1_proc_one;
int slot1_proc_two;
int slot1_proc_three;
//int dlsch_td_one;
//int dlsch_td_two;
//int dlsch_td_three;
//int dlsch_td1_one;
//int dlsch_td1_two;
//int dlsch_td1_three;
} threads_t;
#endif /* _THREADS_T_H_ */
...@@ -53,7 +53,7 @@ static void nr_polar_init(t_nrPolar_params * *polarParams, ...@@ -53,7 +53,7 @@ static void nr_polar_init(t_nrPolar_params * *polarParams,
// printf("currentPtr %p (polarParams %p)\n",currentPtr,polarParams); // printf("currentPtr %p (polarParams %p)\n",currentPtr,polarParams);
//Else, initialize and add node to the end of the linked list. //Else, initialize and add node to the end of the linked list.
t_nrPolar_params *newPolarInitNode = malloc(sizeof(t_nrPolar_params)); t_nrPolar_params *newPolarInitNode = calloc(sizeof(t_nrPolar_params),1);
if (newPolarInitNode != NULL) { if (newPolarInitNode != NULL) {
newPolarInitNode->idx = (messageType * messageLength * aggregation_prime); newPolarInitNode->idx = (messageType * messageLength * aggregation_prime);
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment