From ef49960a8a13919961f71b3f7e8a2bb7e422eb62 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux <raphael.defosseux@eurecom.fr> Date: Thu, 3 Dec 2020 18:40:05 +0100 Subject: [PATCH] [CI][Docker] able to deploy/undeploy a monolithic eNB with a compose file: -- The compose file still keeps a few template fields to be replaced -- initial scenario has a full UE attach / traffic test Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr> --- ci-scripts/cls_containerize.py | 171 +++++++++++++++++- ci-scripts/cls_oaicitest.py | 4 +- ci-scripts/main.py | 27 ++- ci-scripts/xml_class_list.yml | 4 +- ...r1_usrp210_band7_deploy_test_05mhz_tm1.xml | 133 ++++++++++++++ .../fr1_enb_mono_fdd_tim/docker-compose.yml | 6 + 6 files changed, 339 insertions(+), 6 deletions(-) create mode 100644 ci-scripts/xml_files/fr1_usrp210_band7_deploy_test_05mhz_tm1.xml diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index a4de458af0..a33e3f4f29 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -75,9 +75,16 @@ class Containerize(): self.imageKind = '' self.eNB_instance = 0 self.eNB_serverId = ['', '', ''] + self.yamlPath = ['', '', ''] + self.eNB_logFile = ['', '', ''] + self.testCase_id = '' + + self.flexranCtrlDeployed = False + self.flexranCtrlIpAddress = '' self.htmlObj = None self.epcObj = None + self.ranObj = None #----------------------------------------------------------- # Container management functions @@ -102,7 +109,6 @@ class Containerize(): lUserName = self.eNB2UserName lPassWord = self.eNB2Password lSourcePath = self.eNB2SourceCodePath - logging.debug('lIpAddr = ' + lIpAddr + ' lUserName = ' + lUserName + ' lPassWord = ' + lPassWord + ' lSourcePath = ' + lSourcePath) if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -258,3 +264,166 @@ class Containerize(): if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK) + def DeployObject(self): + if self.eNB_serverId[self.eNB_instance] == '0': + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + elif self.eNB_serverId[self.eNB_instance] == '1': + lIpAddr = self.eNB1IPAddress + lUserName = self.eNB1UserName + lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath + elif self.eNB_serverId[self.eNB_instance] == '2': + lIpAddr = self.eNB2IPAddress + lUserName = self.eNB2UserName + lPassWord = self.eNB2Password + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m') + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + # Putting the CPUs in a good state, we do that only on a few servers + mySSH.command('hostname', '\$', 5) + result = re.search('obelix|asterix', mySSH.getBefore()) + if result is not None: + mySSH.command('if command -v cpupower &> /dev/null; then echo ' + lPassWord + ' | sudo -S cpupower idle-set -D 0; fi', '\$', 5) + time.sleep(5) + + mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) + mySSH.command('cp docker-compose.yml ci-docker-compose.yml', '\$', 5) + imageTag = 'develop' + if (self.ranAllowMerge): + imageTag = 'ci-temp' + mySSH.command('sed -i -e "s/image: oai-enb:latest/image: oai-enb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2) + if self.epcObj is not None: + localMmeIpAddr = self.epcObj.MmeIPAddress + mySSH.command('sed -i -e "s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/" ci-docker-compose.yml', '\$', 2) + if self.flexranCtrlDeployed: + mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED:.*/FLEXRAN_ENABLED: "yes"/\' ci-docker-compose.yml', '\$', 2) + mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/" ci-docker-compose.yml', '\$', 2) + else: + mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*$/FLEXRAN_ENABLED: \"no\"/" ci-docker-compose.yml', '\$', 2) + mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/127.0.0.1/" ci-docker-compose.yml', '\$', 2) + # Currently support only one + mySSH.command('docker-compose --file ci-docker-compose.yml config --services | sed -e "s@^@service=@"', '\$', 2) + result = re.search('service=(?P<svc_name>[a-zA-Z0-9\_]+)', mySSH.getBefore()) + if result is not None: + svcName = result.group('svc_name') + mySSH.command('docker-compose --file ci-docker-compose.yml up -d ' + svcName, '\$', 2) + + # Checking Status + mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5) + result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) + unhealthyNb = 0 + healthyNb = 0 + startingNb = 0 + containerName = '' + if result is not None: + containerName = result.group('container_name') + time.sleep(5) + cnt = 0 + while (cnt < 3): + mySSH.command('docker inspect --format=\'{{.State.Health.Status}}\' ' + containerName, '\$', 5) + unhealthyNb = mySSH.getBefore().count('unhealthy') + healthyNb = mySSH.getBefore().count('healthy') - unhealthyNb + startingNb = mySSH.getBefore().count('starting') + if healthyNb == 1: + cnt = 10 + else: + time.sleep(10) + cnt += 1 + logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)') + logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)') + logging.debug(' -- ' + str(startingNb) + ' still starting container(s)') + + status = False + if healthyNb == 1: + cnt = 0 + while (cnt < 20): + mySSH.command('docker logs ' + containerName + ' | egrep --text --color=never -i "wait|sync|Starting"', '\$', 30) + result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore()) + if result is None: + time.sleep(6) + cnt += 1 + else: + cnt = 100 + status = True + logging.info('\u001B[1m Deploying OAI object Pass\u001B[0m') + time.sleep(10) + mySSH.close() + + if self.htmlObj is not None: + self.testCase_id = self.htmlObj.testCase_id + else: + self.testCase_id = '000000' + self.eNB_logFile[self.eNB_instance] = 'enb_' + self.testCase_id + '.log' + + if self.htmlObj is not None: + if status: + self.htmlObj.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + else: + self.htmlObj.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) + + def UndeployObject(self): + logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') + if self.eNB_serverId[self.eNB_instance] == '0': + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + elif self.eNB_serverId[self.eNB_instance] == '1': + lIpAddr = self.eNB1IPAddress + lUserName = self.eNB1UserName + lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath + elif self.eNB_serverId[self.eNB_instance] == '2': + lIpAddr = self.eNB2IPAddress + lUserName = self.eNB2UserName + lPassWord = self.eNB2Password + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m') + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) + # Currently support only one + mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5) + result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) + if result is not None: + containerName = result.group('container_name') + mySSH.command('docker kill --signal INT ' + containerName, '\$', 30) + time.sleep(5) + mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30) + mySSH.command('docker rm -f ' + containerName, '\$', 30) + + # Putting the CPUs back in a idle state, we do that only on a few servers + mySSH.command('hostname', '\$', 5) + result = re.search('obelix|asterix', mySSH.getBefore()) + if result is not None: + mySSH.command('if command -v cpupower &> /dev/null; then echo ' + lPassWord + ' | sudo -S cpupower idle-set -E; fi', '\$', 5) + mySSH.close() + + # Analyzing log file! + if self.ranObj is not None: + copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '.') + nodeB_prefix = 'e' + if (copyin_res == -1): + if self.htmlObj is not None: + self.htmlObj.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!' + self.htmlObj.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) + else: + logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance]) + logStatus = self.ranObj.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance]) + if (logStatus < 0): + if self.htmlObj is not None: + self.htmlObj.CreateHtmlTestRow(self.ranObj.runtime_stats, 'KO', logStatus) + else: + if self.htmlObj is not None: + self.htmlObj.CreateHtmlTestRow(self.ranObj.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) + diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 8b4c292e20..9704b74c2e 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -216,7 +216,7 @@ class OaiCiTest(): HTML.CreateHtmlTabFooter(False) self.ConditionalExit() - def CheckFlexranCtrlInstallation(self,RAN,EPC): + def CheckFlexranCtrlInstallation(self,RAN,EPC,CONTAINERS): if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '': return SSH = sshconnection.SSHConnection() @@ -234,6 +234,8 @@ class OaiCiTest(): if result is not None: RAN.flexranCtrlDeployed=True RAN.flexranCtrlIpAddress=result.group('flex_ip_addr') + CONTAINERS.flexranCtrlDeployed=True + CONTAINERS.flexranCtrlIpAddress=result.group('flex_ip_addr') logging.debug('Flexran Controller is deployed: ' + RAN.flexranCtrlIpAddress) SSH.close() diff --git a/ci-scripts/main.py b/ci-scripts/main.py index b7583ff753..825356a63c 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -310,6 +310,22 @@ def GetParametersFromXML(action): if (string_field is not None): EPC.yamlPath = string_field + elif action == 'Deploy_Object' or action == 'Undeploy_Object': + eNB_instance=test.findtext('eNB_instance') + if (eNB_instance is None): + CONTAINERS.eNB_instance=0 + else: + CONTAINERS.eNB_instance=int(eNB_instance) + eNB_serverId=test.findtext('eNB_serverId') + if (eNB_serverId is None): + CONTAINERS.eNB_serverId[CONTAINERS.eNB_instance]='0' + else: + CONTAINERS.eNB_serverId[CONTAINERS.eNB_instance]=eNB_serverId + string_field = test.findtext('yaml_path') + if (string_field is not None): + CONTAINERS.yamlPath[CONTAINERS.eNB_instance] = string_field + + else: # ie action == 'Run_PhySim': ldpc.runargs = test.findtext('physim_run_args') @@ -368,6 +384,7 @@ RAN.htmlObj=HTML RAN.epcObj=EPC CONTAINERS.htmlObj=HTML CONTAINERS.epcObj=EPC +CONTAINERS.ranObj=RAN ldpc=cls_physim.PhySim() #create an instance for LDPC test using GPU or CPU build @@ -591,7 +608,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re logging.debug('ERROR: requested test is invalidly formatted: ' + test) sys.exit(1) if (EPC.IPAddress != '') and (EPC.IPAddress != 'none'): - CiTestObj.CheckFlexranCtrlInstallation(RAN,EPC) + CiTestObj.CheckFlexranCtrlInstallation(RAN,EPC,CONTAINERS) EPC.SetMmeIPAddress() #get the list of tests to be done @@ -652,8 +669,6 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re break if action == 'Build_eNB': RAN.BuildeNB() - elif action == 'Build_Image': - CONTAINERS.BuildImage() elif action == 'WaitEndBuild_eNB': RAN.WaitBuildeNBisFinished() elif action == 'Initialize_eNB': @@ -729,6 +744,12 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re if ldpc.exitStatus==1:sys.exit() elif action == 'Run_PhySim': HTML=ldpc.Run_PhySim(HTML,CONST,id) + elif action == 'Build_Image': + CONTAINERS.BuildImage() + elif action == 'Deploy_Object': + CONTAINERS.DeployObject() + elif action == 'Undeploy_Object': + CONTAINERS.UndeployObject() else: sys.exit('Invalid class (action) from xml') if not RAN.prematureExit: diff --git a/ci-scripts/xml_class_list.yml b/ci-scripts/xml_class_list.yml index 66c9682ba6..0e00a2e7d4 100755 --- a/ci-scripts/xml_class_list.yml +++ b/ci-scripts/xml_class_list.yml @@ -1,7 +1,6 @@ - Build_PhySim - Run_PhySim - Build_eNB - - Build_Image - WaitEndBuild_eNB - Initialize_eNB - Terminate_eNB @@ -35,3 +34,6 @@ - Ping_CatM_module - IdleSleep - Perform_X2_Handover + - Build_Image + - Deploy_Object + - Undeploy_Object diff --git a/ci-scripts/xml_files/fr1_usrp210_band7_deploy_test_05mhz_tm1.xml b/ci-scripts/xml_files/fr1_usrp210_band7_deploy_test_05mhz_tm1.xml new file mode 100644 index 0000000000..3d04fdeb29 --- /dev/null +++ b/ci-scripts/xml_files/fr1_usrp210_band7_deploy_test_05mhz_tm1.xml @@ -0,0 +1,133 @@ +<!-- + + Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The OpenAirInterface Software Alliance licenses this file to You under + the OAI Public License, Version 1.1 (the "License"); you may not use this file + except in compliance with the License. + You may obtain a copy of the License at + + http://www.openairinterface.org/?page_id=698 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + For more information about the OpenAirInterface (OAI) Software Alliance: + contact@openairinterface.org + +--> +<testCaseList> + <htmlTabRef>test-deploy-enb-mono</htmlTabRef> + <htmlTabName>Test-Deploy-eNB-Mono</htmlTabName> + <htmlTabIcon>tasks</htmlTabIcon> + <repeatCount>1</repeatCount> + <TestCaseRequestedList> + 040101 + 030101 000020 040301 000021 040501 040601 040611 040641 040651 040401 040201 030201 + </TestCaseRequestedList> + <TestCaseExclusionList></TestCaseExclusionList> + + <testCase id="000001"> + <class>IdleSleep</class> + <desc>Waiting for 60 seconds</desc> + <idle_sleep_time_in_sec>60</idle_sleep_time_in_sec> + </testCase> + + <testCase id="000002"> + <class>IdleSleep</class> + <desc>Waiting for 10 seconds</desc> + <idle_sleep_time_in_sec>10</idle_sleep_time_in_sec> + </testCase> + + <testCase id="000020"> + <class>CheckStatusUE</class> + <desc>Check UE(s) status before attachment</desc> + <expectedNbOfConnectedUEs>0</expectedNbOfConnectedUEs> + </testCase> + + <testCase id="000021"> + <class>CheckStatusUE</class> + <desc>Check UE(s) status after attachment</desc> + <expectedNbOfConnectedUEs>1</expectedNbOfConnectedUEs> + </testCase> + + <testCase id="030101"> + <class>Deploy_Object</class> + <desc>Deploy eNB (FDD/Band7/5MHz) in a container</desc> + <yaml_path>ci-scripts/yaml_files/fr1_enb_mono_fdd_tim</yaml_path> + <eNB_instance>0</eNB_instance> + <eNB_serverId>0</eNB_serverId> + </testCase> + + <testCase id="030201"> + <class>Undeploy_Object</class> + <desc>Undeploy eNB</desc> + <yaml_path>ci-scripts/yaml_files/fr1_enb_mono_fdd_tim</yaml_path> + <eNB_instance>0</eNB_instance> + <eNB_serverId>0</eNB_serverId> + </testCase> + + <testCase id="040101"> + <class>Initialize_UE</class> + <desc>Initialize UE</desc> + </testCase> + + <testCase id="040201"> + <class>Terminate_UE</class> + <desc>Terminate UE</desc> + </testCase> + + <testCase id="040301"> + <class>Attach_UE</class> + <desc>Attach UE</desc> + </testCase> + + <testCase id="040401"> + <class>Detach_UE</class> + <desc>Detach UE</desc> + </testCase> + + <testCase id="040501"> + <class>Ping</class> + <desc>ping (5MHz - 20 sec)</desc> + <ping_args>-c 20</ping_args> + <ping_packetloss_threshold>5</ping_packetloss_threshold> + </testCase> + + <testCase id="040601"> + <class>Iperf</class> + <desc>iperf (5MHz - DL/15Mbps/UDP)(30 sec)</desc> + <iperf_args>-u -b 15M -t 30 -i 1</iperf_args> + <iperf_packetloss_threshold>50</iperf_packetloss_threshold> + <iperf_profile>single-ue</iperf_profile> + </testCase> + + <testCase id="040611"> + <class>Iperf</class> + <desc>iperf (5MHz - DL/TCP)(30 sec)</desc> + <iperf_args>-t 30 -i 1</iperf_args> + <iperf_packetloss_threshold>50</iperf_packetloss_threshold> + <iperf_profile>single-ue</iperf_profile> + </testCase> + + <testCase id="040641"> + <class>Iperf</class> + <desc>iperf (5MHz - UL/7.5Mbps/UDP)(30 sec)</desc> + <iperf_args>-u -b 7.5M -t 30 -i 1 -R</iperf_args> + <iperf_packetloss_threshold>50</iperf_packetloss_threshold> + <iperf_profile>single-ue</iperf_profile> + </testCase> + + <testCase id="040651"> + <class>Iperf</class> + <desc>iperf (5MHz - UL/TCP)(30 sec)</desc> + <iperf_args>-t 30 -i 1 -R</iperf_args> + <iperf_packetloss_threshold>50</iperf_packetloss_threshold> + <iperf_profile>single-ue</iperf_profile> + </testCase> + +</testCaseList> diff --git a/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml b/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml index 21c88bc1ab..de52811435 100644 --- a/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml +++ b/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml @@ -33,6 +33,12 @@ services: networks: public_net: ipv4_address: 192.168.61.30 + healthcheck: + # pgrep does NOT work + test: /bin/bash -c "ps aux | grep -v grep | grep -c softmodem" + interval: 10s + timeout: 5s + retries: 5 networks: public_net: -- 2.26.2