fix(ci): 4g/5g RF simulation tests may fail due to lost socket on ue...

fix(ci): 4g/5g RF simulation tests may fail due to lost socket on ue container. Repeat in case of failure
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>

ci-scripts/cls_containerize.py

@@ -722,7 +722,7 @@ class Containerize():
 		cmd = 'mkdir -p ../cmake_targets/log'
 		deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
 
-		cmd = 'docker exec ' + self.pingContName + ' /bin/bash -c "ping ' + self.pingOptions + '" 2>&1 | tee ../cmake_targets/log/ping_' + HTML.testCase_id + '.log'
+		cmd = 'docker exec ' + self.pingContName + ' /bin/bash -c "ping ' + self.pingOptions + '" 2>&1 | tee ../cmake_targets/log/ping_' + HTML.testCase_id + '.log || true'
 		logging.debug(cmd)
 		deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100)
 
@@ -788,18 +788,18 @@ class Containerize():
 		logStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
 
 		# Start the server process
-		cmd = 'docker exec -d ' + self.svrContName + ' /bin/bash -c "nohup iperf ' + self.svrOptions + ' > /tmp/iperf_server.log 2>&1"'
+		cmd = 'docker exec -d ' + self.svrContName + ' /bin/bash -c "nohup iperf ' + self.svrOptions + ' > /tmp/iperf_server.log 2>&1" || true'
 		logging.debug(cmd)
 		serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
 		time.sleep(5)
 
 		# Start the client process
-		cmd = 'docker exec ' + self.cliContName + ' /bin/bash -c "iperf ' + self.cliOptions + '" 2>&1 | tee ../cmake_targets/log/iperf_client_' + HTML.testCase_id + '.log'
+		cmd = 'docker exec ' + self.cliContName + ' /bin/bash -c "iperf ' + self.cliOptions + '" 2>&1 | tee ../cmake_targets/log/iperf_client_' + HTML.testCase_id + '.log || true'
 		logging.debug(cmd)
 		clientStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100)
 
 		# Stop the server process
-		cmd = 'docker exec ' + self.svrContName + ' /bin/bash -c "pkill iperf"'
+		cmd = 'docker exec ' + self.svrContName + ' /bin/bash -c "pkill iperf" || true'
 		logging.debug(cmd)
 		serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
 		time.sleep(5)
@@ -816,6 +816,7 @@ class Containerize():
 			else:
 				message = 'Server Report and Connection refused Not Found!'
 			self.IperfExit(HTML, False, message)
+			logging.error('\u001B[1;37;41m Iperf Test FAIL\u001B[0m')
 			return
 
 		# Computing the requested bandwidth in float
@@ -888,6 +889,8 @@ class Containerize():
 			self.IperfExit(HTML, iperfStatus, 'problem?')
 		if iperfStatus:
 			logging.info('\u001B[1m Iperf Test PASS\u001B[0m')
+		else:
+			logging.error('\u001B[1;37;41m Iperf Test FAIL\u001B[0m')
 
 	def IperfExit(self, HTML, status, message):
 		html_queue = SimpleQueue()

ci-scripts/xml_files/container_4g_rfsim.xml

@@ -24,6 +24,7 @@
         <htmlTabRef>rfsim-4glte</htmlTabRef>
         <htmlTabName>Testing 4G LTE RF sim in containers</htmlTabName>
         <htmlTabIcon>wrench</htmlTabIcon>
+        <repeatCount>2</repeatCount>
         <TestCaseRequestedList>
  100011
  000011

ci-scripts/xml_files/container_5g_rfsim.xml

@@ -24,6 +24,7 @@
         <htmlTabRef>rfsim-5gnr</htmlTabRef>
         <htmlTabName>Testing 5G NR RF sim in containers</htmlTabName>
         <htmlTabIcon>wrench</htmlTabIcon>
+        <repeatCount>4</repeatCount>
         <TestCaseRequestedList>
  100001
  000001

docker/scripts/enb_entrypoint.sh

@@ -9,15 +9,15 @@ ENABLE_X2=${ENABLE_X2:-no}
 THREAD_PARALLEL_CONFIG=${THREAD_PARALLEL_CONFIG:-PARALLEL_SINGLE_THREAD}
 
 # Based another env var, pick one template to use
-if [[ -v USE_FDD_CU ]]; then ln -s $PREFIX/etc/cu.fdd.conf $PREFIX/etc/enb.conf; fi
-if [[ -v USE_FDD_DU ]]; then ln -s $PREFIX/etc/du.fdd.conf $PREFIX/etc/enb.conf; fi
-if [[ -v USE_FDD_MONO ]]; then ln -s $PREFIX/etc/enb.fdd.conf $PREFIX/etc/enb.conf; fi
-if [[ -v USE_TDD_MONO ]]; then ln -s $PREFIX/etc/enb.tdd.conf $PREFIX/etc/enb.conf; fi
-if [[ -v USE_FDD_FAPI_RCC ]]; then ln -s $PREFIX/etc/rcc.nfapi.fdd.conf $PREFIX/etc/enb.conf; fi
-if [[ -v USE_FDD_IF4P5_RCC ]]; then ln -s $PREFIX/etc/rcc.if4p5.fdd.conf $PREFIX/etc/enb.conf; fi
-if [[ -v USE_TDD_IF4P5_RCC ]]; then ln -s $PREFIX/etc/rcc.if4p5.tdd.conf $PREFIX/etc/enb.conf; fi
-if [[ -v USE_FDD_RRU ]]; then ln -s $PREFIX/etc/rru.fdd.conf $PREFIX/etc/enb.conf; fi
-if [[ -v USE_TDD_RRU ]]; then ln -s $PREFIX/etc/rru.tdd.conf $PREFIX/etc/enb.conf; fi
+if [[ -v USE_FDD_CU ]]; then cp $PREFIX/etc/cu.fdd.conf $PREFIX/etc/enb.conf; fi
+if [[ -v USE_FDD_DU ]]; then cp $PREFIX/etc/du.fdd.conf $PREFIX/etc/enb.conf; fi
+if [[ -v USE_FDD_MONO ]]; then cp $PREFIX/etc/enb.fdd.conf $PREFIX/etc/enb.conf; fi
+if [[ -v USE_TDD_MONO ]]; then cp $PREFIX/etc/enb.tdd.conf $PREFIX/etc/enb.conf; fi
+if [[ -v USE_FDD_FAPI_RCC ]]; then cp $PREFIX/etc/rcc.nfapi.fdd.conf $PREFIX/etc/enb.conf; fi
+if [[ -v USE_FDD_IF4P5_RCC ]]; then cp $PREFIX/etc/rcc.if4p5.fdd.conf $PREFIX/etc/enb.conf; fi
+if [[ -v USE_TDD_IF4P5_RCC ]]; then cp $PREFIX/etc/rcc.if4p5.tdd.conf $PREFIX/etc/enb.conf; fi
+if [[ -v USE_FDD_RRU ]]; then cp $PREFIX/etc/rru.fdd.conf $PREFIX/etc/enb.conf; fi
+if [[ -v USE_TDD_RRU ]]; then cp $PREFIX/etc/rru.tdd.conf $PREFIX/etc/enb.conf; fi
 
 # Only this template will be manipulated
 CONFIG_FILES=`ls $PREFIX/etc/enb.conf || true`

docker/scripts/gnb_entrypoint.sh

@@ -7,8 +7,8 @@ ENABLE_X2=${ENABLE_X2:-yes}
 THREAD_PARALLEL_CONFIG=${THREAD_PARALLEL_CONFIG:-PARALLEL_SINGLE_THREAD}
 
 # Based another env var, pick one template to use
-if [[ -v USE_NSA_TDD_MONO ]]; then ln -s $PREFIX/etc/gnb.nsa.tdd.conf $PREFIX/etc/gnb.conf; fi
-if [[ -v USE_SA_TDD_MONO ]]; then ln -s $PREFIX/etc/gnb.sa.tdd.conf $PREFIX/etc/gnb.conf; fi
+if [[ -v USE_NSA_TDD_MONO ]]; then cp $PREFIX/etc/gnb.nsa.tdd.conf $PREFIX/etc/gnb.conf; fi
+if [[ -v USE_SA_TDD_MONO ]]; then cp $PREFIX/etc/gnb.sa.tdd.conf $PREFIX/etc/gnb.conf; fi
 
 # Only this template will be manipulated
 CONFIG_FILES=`ls $PREFIX/etc/gnb.conf || true`

docker/scripts/lte_ru_entrypoint.sh

@@ -5,8 +5,8 @@ set -euo pipefail
 PREFIX=/opt/oai-lte-ru
 
 # Based another env var, pick one template to use
-if [[ -v USE_FDD_RRU ]]; then ln -s $PREFIX/etc/rru.fdd.conf $PREFIX/etc/rru.conf; fi
-if [[ -v USE_TDD_RRU ]]; then ln -s $PREFIX/etc/rru.tdd.conf $PREFIX/etc/rru.conf; fi
+if [[ -v USE_FDD_RRU ]]; then cp $PREFIX/etc/rru.fdd.conf $PREFIX/etc/rru.conf; fi
+if [[ -v USE_TDD_RRU ]]; then cp $PREFIX/etc/rru.tdd.conf $PREFIX/etc/rru.conf; fi
 
 # Only this template will be manipulated
 CONFIG_FILES=`ls $PREFIX/etc/rru.conf || true`

docker/scripts/lte_ue_entrypoint.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 PREFIX=/opt/oai-lte-ue
 
 # Based another env var, pick one template to use
-if [[ -v USE_NFAPI ]]; then ln -s $PREFIX/etc/ue.nfapi.conf $PREFIX/etc/ue.conf; fi
+if [[ -v USE_NFAPI ]]; then cp $PREFIX/etc/ue.nfapi.conf $PREFIX/etc/ue.conf; fi
 
 # Only this template will be manipulated and the USIM one!
 CONFIG_FILES=`ls $PREFIX/etc/ue.conf $PREFIX/etc/ue_usim.conf || true`

docker/scripts/nr_ue_entrypoint.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 PREFIX=/opt/oai-nr-ue
 
 # Based another env var, pick one template to use
-#if [[ -v USE_NFAPI ]]; then ln -s $PREFIX/etc/ue.nfapi.conf $PREFIX/etc/ue.conf; fi
+#if [[ -v USE_NFAPI ]]; then cp $PREFIX/etc/ue.nfapi.conf $PREFIX/etc/ue.conf; fi
 
 # Only this template will be manipulated and the USIM one!
 CONFIG_FILES=`ls $PREFIX/etc/ue.conf $PREFIX/etc/nr-ue-sim.conf || true`