Integration 2021 wk23

MR !1180: \[CI\] nsa traces 
-NSA test and pipeline update for UE-module
-improve traces and log collection (pcap, QLog, stats)
 
MR !1126: bugfix-nr-t-reordering 
-deal with "infinity" case for t-reordering
-some small fixes (one bug i pdcp, one log set to LOG_E in rlc)
 
MR !1168: nr-pdcp-small-bugfixes
-better nettle version detection
-bad HFN computation at PDU reception

ci-scripts/Jenkinsfile-tmp-multi-enb-nsa

@@ -33,10 +33,9 @@ def buildStageStatus = true
 // Name of the test stage
 def testStageName = params.pipelineTestStageName
 
-// Name of the phone resource
+// Name of the phone/server resource
 def ciSmartPhonesResource1 = params.SmartPhonesResource1
 def ciSmartPhonesResource2 = params.SmartPhonesResource2
-def ciSmartPhonesResource3 = params.SmartPhonesResource3
 
 // Global Parameters. Normally they should be populated when the master job
 // triggers the slave job with parameters
@@ -53,7 +52,7 @@ pipeline {
     options {
         disableConcurrentBuilds()
         ansiColor('xterm')
-        lock(extra: [[resource: ciSmartPhonesResource2],[resource: ciSmartPhonesResource3]], resource: ciSmartPhonesResource1)
+        lock(extra: [[resource: ciSmartPhonesResource2]], resource: ciSmartPhonesResource1)
     }
     stages {
         stage("Build Init") {
@@ -85,9 +84,6 @@ pipeline {
                     if (params.SmartPhonesResource2 == null) {
                         allParametersPresent = false
                     }
-                    if (params.SmartPhonesResource3== null) {
-                        allParametersPresent = false
-                    }
                     // 1st eNB parameters
                     if (params.eNB_IPAddress == null) {
                         allParametersPresent = false

ci-scripts/Jenkinsfile-trig-nsa

@@ -36,7 +36,7 @@ pipeline {
         stage ("Launcher") {
             steps {
                 script {     
-                    //retrieve MR that are opened nd with tag READY_TO_BE_MERGED             
+                    //retrieve MR that are opened nd with tag NSA             
                     MR_LIST= sh returnStdout: true, script: 'curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100&labels=NSA" | jq ".[].iid" || true '
                     echo "List of selected MR:\n${MR_LIST}" 
                     def MR_ARRAY = MR_LIST.split('\n') 
@@ -48,7 +48,7 @@ pipeline {
                         COMMIT_ID=COMMIT_ID.trim()
                         echo "Testing NSA on : ${MR} ${SRC_BRANCH} ${COMMIT_ID}"
                         //calling NSA sub job
-                        build job: "RAN-CI-NSA-B210-N310-ModuleUE", wait : false, propagate : false, parameters: [
+                        build job: "RAN-NSA-Mini-Module", wait : false, propagate : false, parameters: [
                             string(name: 'eNB_MR', value: String.valueOf(MR)),                          
                             string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)),
                             string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)),
@@ -56,13 +56,13 @@ pipeline {
                             booleanParam(name: 'eNB_AllowMergeRequestProcess', value: Boolean.valueOf(ALLOW_MERGE))
                         ]
                         //calling Benetel sub job
-                        build job: "RAN-CI-BENETEL", wait : false, propagate : false, parameters: [
-                            string(name: 'eNB_MR', value: String.valueOf(MR)),                          
-                            string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)),
-                            string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)),
-                            string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)),
-                            booleanParam(name: 'eNB_AllowMergeRequestProcess', value: Boolean.valueOf(ALLOW_MERGE))
-                        ]
+                        //build job: "RAN-CI-BENETEL", wait : false, propagate : false, parameters: [
+                        //    string(name: 'eNB_MR', value: String.valueOf(MR)),                          
+                        //    string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)),
+                        //    string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)),
+                        //    string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)),
+                        //    booleanParam(name: 'eNB_AllowMergeRequestProcess', value: Boolean.valueOf(ALLOW_MERGE))
+                        //]
                     }
                 }                                  
             }   

ci-scripts/ci_ueinfra.yaml

@@ -8,7 +8,7 @@ idefix:
   WakeupScript : ci_ctl_qtel.py /dev/ttyUSB2 wup
   DetachScript : ci_ctl_qtel.py /dev/ttyUSB2 detach
   PLMN : 22201
-  UENetwork : enxb22a941dbf00 
+  UENetwork : wwan0 
   HostIPAddress : 192.168.18.188
   HostUsername : oaicicd
   HostPassword : oaicicd

ci-scripts/cls_module_ue.py

@@ -37,7 +37,7 @@ import time
 import re
 import subprocess
 
-
+from datetime import datetime
 
 class Module_UE:
 
@@ -101,7 +101,7 @@ class Module_UE:
 	def GetModuleIPAddress(self):
 		HOST=self.HostIPAddress
 		response= []
-		tentative = 10
+		tentative = 3 
 		while (len(response)==0) and (tentative>0):
 			COMMAND="ip a show dev " + self.UENetwork + " | grep inet | grep " + self.UENetwork
 			logging.debug(COMMAND)
@@ -126,7 +126,37 @@ class Module_UE:
 				logging.debug('\u001B[1;37;41m Module IP Address Not Found! \u001B[0m')
 				return -1
 
+	def EnableTrace(self):
+		mySSH = sshconnection.SSHConnection()
+		mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword)
+		#delete old artifacts
+		mySSH.command('echo ' + self.HostPassword + ' | sudo -S rm -rf ci_qlog','\$',5)
+		#start Trace
+		mySSH.command('echo $USER; nohup sudo -E QLog/QLog -s ci_qlog -f NR5G.cfg &','\$', 5)
+		mySSH.close()
+
+	def DisableTrace(self):
+		mySSH = sshconnection.SSHConnection()
+		mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword)
+		mySSH.command('echo ' + self.HostPassword + ' | sudo -S killall --signal=SIGINT *QLog*', '\$',5)
+		mySSH.close()
 
 
+	def DisableCM(self):
+		mySSH = sshconnection.SSHConnection()
+		mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword)
+		mySSH.command('echo ' + self.HostPassword + ' | sudo -S killall --signal SIGKILL *'+self.Process['Name']+'*', '\$', 5)
+		mySSH.close()
 
 
+	def LogCollect(self):
+		mySSH = sshconnection.SSHConnection()
+		mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword)
+		#archive qlog to /opt/ci_qlogs with datetime suffix
+		now=datetime.now()
+		now_string = now.strftime("%Y%m%d-%H%M")
+		source='ci_qlog'
+		destination='/opt/ci_qlogs/ci_qlog_'+now_string+'.zip'
+		mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S zip -r '+destination+' '+source+' &','\$', 10)
+		mySSH.close()
+		return destination

ci-scripts/cls_oaicitest.py

@@ -388,18 +388,23 @@ class OaiCiTest():
 			Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id])
 			is_module=Module_UE.CheckCMProcess()
 			if is_module:
+				Module_UE.EnableTrace()
+				time.sleep(5)
 				Module_UE.Command("wup")
+				logging.debug("Waiting for IP address to be assigned")
+				time.sleep(20)
+				logging.debug("Retrieve IP address")
 				status=Module_UE.GetModuleIPAddress()
 				if status==0:
 					HTML.CreateHtmlTestRow(Module_UE.UEIPAddress, 'OK', CONST.ALL_PROCESSES_OK)	
 					logging.debug('UE IP addresss : '+ Module_UE.UEIPAddress)
 				else: #status==-1 failed to retrieve IP address
 					HTML.CreateHtmlTestRow('N/A', 'KO', CONST.UE_IP_ADDRESS_ISSUE)
-					self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+					self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 					return
 
 
-	def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE):
+	def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE,InfraUE):
 		if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
 			HELP.GenericHelp(CONST.Version)
 			sys.exit('Insufficient Parameter')
@@ -623,7 +628,7 @@ class OaiCiTest():
 				HTML.htmlUEFailureMsg='nr-uesoftmodem did NOT synced'
 				HTML.CreateHtmlTestRow(self.air_interface + ' ' +  self.Initialize_OAI_UE_args, 'KO', CONST.OAI_UE_PROCESS_COULD_NOT_SYNC, 'OAI UE')
 			logging.error('\033[91mInitialize OAI UE Failed! \033[0m')
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 
 	def checkDevTTYisUnlocked(self):
 		SSH = sshconnection.SSHConnection()
@@ -701,7 +706,7 @@ class OaiCiTest():
 		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
 		self.checkDevTTYisUnlocked()
 
-	def AttachCatM(self,HTML,RAN,COTS_UE,EPC):
+	def AttachCatM(self,HTML,RAN,COTS_UE,EPC,InfraUE):
 		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
 			HELP.GenericHelp(CONST.Version)
 			sys.exit('Insufficient Parameter')
@@ -774,9 +779,9 @@ class OaiCiTest():
 			html_cell = '<pre style="background-color:white">CAT-M module Attachment Failed</pre>'
 			html_queue.put(html_cell)
 			HTML.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 
-	def PingCatM(self,HTML,RAN,EPC,COTS_UE):
+	def PingCatM(self,HTML,RAN,EPC,COTS_UE,InfraUE):
 		if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '':
 			HELP.GenericHelp(CONST.Version)
 			sys.exit('Insufficient Parameter')
@@ -785,7 +790,7 @@ class OaiCiTest():
 		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
 		if (pStatus < 0):
 			HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 			return
 		try:
 			statusQueue = SimpleQueue()
@@ -806,7 +811,7 @@ class OaiCiTest():
 					moduleIPAddr = result.group('ipaddr')
 				else:
 					HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
-					self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+					self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 					return
 			ping_time = re.findall("-c (\d+)",str(self.ping_args))
 			device_id = 'catm'
@@ -870,7 +875,7 @@ class OaiCiTest():
 				HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', 1, statusQueue)
 			else:
 				HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue)
-				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 		except:
 			os.kill(os.getppid(),signal.SIGUSR1)
 
@@ -962,7 +967,7 @@ class OaiCiTest():
 		except:
 			os.kill(os.getppid(),signal.SIGUSR1)
 
-	def AttachUE(self,HTML,RAN,EPC,COTS_UE):
+	def AttachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE):
 		if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB
 			if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
 				HELP.GenericHelp(CONST.Version)
@@ -972,7 +977,7 @@ class OaiCiTest():
 			pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
 			if (pStatus < 0):
 				HTML.CreateHtmlTestRow('N/A', 'KO', pStatus)
-				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 				return
 			multi_jobs = []
 			status_queue = SimpleQueue()
@@ -991,7 +996,7 @@ class OaiCiTest():
 
 		if (status_queue.empty()):
 			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 			return
 		else:
 			attach_status = True
@@ -1020,7 +1025,7 @@ class OaiCiTest():
 					time.sleep(5)
 			else:
 				HTML.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
-				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 
 	def DetachUE_common(self, device_id, idx,COTS_UE):
 		try:
@@ -1054,7 +1059,7 @@ class OaiCiTest():
 			pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
 			if (pStatus < 0):
 				HTML.CreateHtmlTestRow('N/A', 'KO', pStatus)
-				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 				return
 			multi_jobs = []
 			cnt = 0
@@ -1079,7 +1084,10 @@ class OaiCiTest():
 		else:#if an ID is specified, it is a module from the yaml infrastructure file
 			Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id])
 			Module_UE.Command("detach")
-			HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)	
+			Module_UE.DisableTrace()
+			Module_UE.DisableCM()
+			archive_destination=Module_UE.LogCollect()
+			HTML.CreateHtmlTestRow('QLog at : '+archive_destination, 'OK', CONST.ALL_PROCESSES_OK)	
 				
 							
 
@@ -1329,7 +1337,7 @@ class OaiCiTest():
 		except:
 			os.kill(os.getppid(),signal.SIGUSR1)
 
-	def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE):
+	def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE,InfraUE):
 		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
 			HELP.GenericHelp(CONST.Version)
 			sys.exit('Insufficient Parameter')
@@ -1376,7 +1384,7 @@ class OaiCiTest():
 
 		if (status_queue.empty()):
 			HTML.CreateHtmlTestRow(htmlOptions, 'KO', CONST.ALL_PROCESSES_OK)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 		else:
 			check_status = True
 			html_queue = SimpleQueue()
@@ -1392,7 +1400,7 @@ class OaiCiTest():
 				HTML.CreateHtmlTestRowQueue(htmlOptions, 'OK', len(self.UEDevices), html_queue)
 			else:
 				HTML.CreateHtmlTestRowQueue(htmlOptions, 'KO', len(self.UEDevices), html_queue)
-				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 
 	def GetAllUEIPAddresses(self):
 		SSH = sshconnection.SSHConnection()
@@ -1612,14 +1620,14 @@ class OaiCiTest():
 		html_queue.put(html_cell)
 		HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
 
-	def PingNoS1(self,HTML,RAN,EPC,COTS_UE):
+	def PingNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE):
 		SSH=sshconnection.SSHConnection()
 		check_eNB = True
 		check_OAI_UE = True
 		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
 		if (pStatus < 0):
 			HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 			return
 		ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args))
 		if ping_from_eNB is not None:
@@ -1708,7 +1716,7 @@ class OaiCiTest():
 	def Ping(self,HTML,RAN,EPC,COTS_UE, InfraUE):
 		result = re.search('noS1', str(RAN.Initialize_eNB_args))
 		if result is not None:
-			self.PingNoS1(HTML,RAN,EPC,COTS_UE)
+			self.PingNoS1(HTML,RAN,EPC,COTS_UE,InfraUE)
 			return
 		if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '':
 			HELP.GenericHelp(CONST.Version)
@@ -1721,7 +1729,7 @@ class OaiCiTest():
 		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
 		if (pStatus < 0):
 			HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 			return
 
 		if self.ue_id=="":
@@ -1729,7 +1737,7 @@ class OaiCiTest():
 			ueIpStatus = self.GetAllUEIPAddresses()
 			if (ueIpStatus < 0):
 				HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE)
-				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 				return
 		else:
 			self.UEIPAddresses=[]
@@ -1756,7 +1764,7 @@ class OaiCiTest():
 
 		if (status_queue.empty()):
 			HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.ALL_PROCESSES_OK)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 		else:
 			ping_status = True
 			html_queue = SimpleQueue()
@@ -1773,7 +1781,7 @@ class OaiCiTest():
 				HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue)
 			else:
 				HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
-				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 
 	def Iperf_ComputeTime(self):
 		result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args))
@@ -2467,7 +2475,7 @@ class OaiCiTest():
 		except:
 			os.kill(os.getppid(),signal.SIGUSR1)
 
-	def IperfNoS1(self,HTML,RAN,EPC,COTS_UE):
+	def IperfNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE):
 		SSH = sshconnection.SSHConnection()
 		if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
 			HELP.GenericHelp(CONST.Version)
@@ -2477,7 +2485,7 @@ class OaiCiTest():
 		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
 		if (pStatus < 0):
 			HTML.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 			return
 		server_on_enb = re.search('-R', str(self.iperf_args))
 		if server_on_enb is not None:
@@ -2576,12 +2584,12 @@ class OaiCiTest():
 			HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue)
 		else:
 			HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 
 	def Iperf(self,HTML,RAN,EPC,COTS_UE, InfraUE):
 		result = re.search('noS1', str(RAN.Initialize_eNB_args))
 		if result is not None:
-			self.IperfNoS1(HTML,RAN,EPC,COTS_UE)
+			self.IperfNoS1(HTML,RAN,EPC,COTS_UE,InfraUE)
 			return
 		if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '' or self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
 			HELP.GenericHelp(CONST.Version)
@@ -2594,14 +2602,14 @@ class OaiCiTest():
 		pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
 		if (pStatus < 0):
 			HTML.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 			return
 
 		if self.ue_id=="":#is not a module, follow legacy code
 			ueIpStatus = self.GetAllUEIPAddresses()
 			if (ueIpStatus < 0):
 				HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE)
-				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 				return
 		else: #is a module
 			self.UEIPAddresses=[]
@@ -2646,7 +2654,7 @@ class OaiCiTest():
 
 		if (status_queue.empty()):
 			HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.ALL_PROCESSES_OK)
-			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+			self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfaUE)
 		else:
 			iperf_status = True
 			iperf_noperf = False
@@ -2668,7 +2676,7 @@ class OaiCiTest():
 				HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue)
 			else:
 				HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
-				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+				self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 
 	def CheckProcessExist(self, check_eNB, check_OAI_UE,RAN,EPC):
 		multi_jobs = []
@@ -3088,7 +3096,7 @@ class OaiCiTest():
 			job.join()
 		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
 
-	def TerminateOAIUE(self,HTML,RAN,COTS_UE,EPC):
+	def TerminateOAIUE(self,HTML,RAN,COTS_UE,EPC, InfraUE):
 		SSH = sshconnection.SSHConnection()
 		SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
 		SSH.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5)
@@ -3129,11 +3137,11 @@ class OaiCiTest():
 					# Not an error then
 					if (logStatus != CONST.OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'):
 						self.Initialize_OAI_UE_args = ''
-						self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+						self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 				else:
 					if (logStatus == CONST.OAI_UE_PROCESS_COULD_NOT_SYNC):
 						self.Initialize_OAI_UE_args = ''
-						self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC)
+						self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
 			else:
 				logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m')
 				HTML.htmlUEFailureMsg='<b>' + ueAction + ' Completed</b>\n' + HTML.htmlUEFailureMsg
@@ -3142,7 +3150,14 @@ class OaiCiTest():
 		else:
 			HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
 
-	def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC):
+	def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC,InfraUE):
+		if self.ue_id!='':
+			Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id])
+			Module_UE.Command("detach")
+			Module_UE.DisableTrace()
+			Module_UE.DisableCM()
+			archive_destination=Module_UE.LogCollect()
+			HTML.CreateHtmlTestRow('QLog at : '+archive_destination, 'OK', CONST.ALL_PROCESSES_OK)	
 		if (self.ADBIPAddress != 'none'):
 			self.testCase_id = 'AUTO-KILL-UE'
 			HTML.testCase_id=self.testCase_id
@@ -3156,7 +3171,7 @@ class OaiCiTest():
 			self.desc = 'Automatic Termination of OAI-UE'
 			HTML.desc='Automatic Termination of OAI-UE'
 			self.ShowTestID()
-			self.TerminateOAIUE(HTML,RAN,COTS_UE,EPC)
+			self.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE)
 		if (RAN.Initialize_eNB_args != ''):
 			self.testCase_id = 'AUTO-KILL-RAN'
 			HTML.testCase_id=self.testCase_id

ci-scripts/main.py

@@ -152,6 +152,7 @@ def GetParametersFromXML(action):
 			RAN.eNB_serverId[RAN.eNB_instance]=eNB_serverId
 
 	elif action == 'Initialize_eNB':
+		RAN.eNB_Trace=test.findtext('eNB_Trace')
 		RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args')
 		eNB_instance=test.findtext('eNB_instance')
 		if (eNB_instance is None):
@@ -474,7 +475,7 @@ elif re.match('^TerminateOAIUE$', mode, re.IGNORECASE):
 		HELP.GenericHelp(CONST.Version)
 		sys.exit('Insufficient Parameter')
 	signal.signal(signal.SIGUSR1, receive_signal)
-	CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC)
+	CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE)
 elif re.match('^TerminateHSS$', mode, re.IGNORECASE):
 	if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.Type == '' or EPC.SourceCodePath == '':
 		HELP.GenericHelp(CONST.Version)
@@ -717,7 +718,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 				elif action == 'Terminate_UE':
 					CiTestObj.TerminateUE(HTML,COTS_UE)
 				elif action == 'Attach_UE':
-					CiTestObj.AttachUE(HTML,RAN,EPC,COTS_UE)
+					CiTestObj.AttachUE(HTML,RAN,EPC,COTS_UE,InfraUE)
 				elif action == 'Detach_UE':
 					CiTestObj.DetachUE(HTML,RAN,EPC,COTS_UE,InfraUE)
 				elif action == 'DataDisable_UE':
@@ -725,23 +726,23 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 				elif action == 'DataEnable_UE':
 					CiTestObj.DataEnableUE(HTML)
 				elif action == 'CheckStatusUE':
-					CiTestObj.CheckStatusUE(HTML,RAN,EPC,COTS_UE)
+					CiTestObj.CheckStatusUE(HTML,RAN,EPC,COTS_UE,InfraUE)
 				elif action == 'Build_OAI_UE':
 					CiTestObj.BuildOAIUE(HTML)
 				elif action == 'Initialize_OAI_UE':
-					CiTestObj.InitializeOAIUE(HTML,RAN,EPC,COTS_UE)
+					CiTestObj.InitializeOAIUE(HTML,RAN,EPC,COTS_UE,InfraUE)
 				elif action == 'Terminate_OAI_UE':
-					CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC)
+					CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE)
 				elif action == 'Initialize_CatM_module':
 					CiTestObj.InitializeCatM(HTML)
 				elif action == 'Terminate_CatM_module':
 					CiTestObj.TerminateCatM(HTML)
 				elif action == 'Attach_CatM_module':
-					CiTestObj.AttachCatM(HTML,RAN,COTS_UE,EPC)
+					CiTestObj.AttachCatM(HTML,RAN,COTS_UE,EPC,InfraUE)
 				elif action == 'Detach_CatM_module':
 					CiTestObj.TerminateCatM(HTML)
 				elif action == 'Ping_CatM_module':
-					CiTestObj.PingCatM(HTML,RAN,EPC,COTS_UE,EPC)
+					CiTestObj.PingCatM(HTML,RAN,EPC,COTS_UE,EPC,InfraUE)
 				elif action == 'Ping':
 					CiTestObj.Ping(HTML,RAN,EPC,COTS_UE, InfraUE)
 				elif action == 'Iperf':

ci-scripts/ran.py

@@ -93,6 +93,7 @@ class RANManagement():
 		self.epcPcapFile = ''
 		self.runtime_stats= ''
 		self.datalog_rt_stats={}
+		self.eNB_Trace = '' #if 'yes', Tshark will be launched at initialization
 
 
 
@@ -344,6 +345,22 @@ class RANManagement():
 			HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus)
 			HTML.CreateHtmlTabFooter(False)
 			sys.exit(1)
+		#Get pcap on S1 and X2 eNB interface, if enabled in the xml 
+		#will not work for gNB at this stage
+		if ((self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb')) and self.eNB_Trace=='yes':
+			mySSH.open(lIpAddr, lUserName, lPassWord)
+			mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
+			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore())
+			if result is not None:
+				eth_interface = result.group('eth_interface')
+				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
+				pcapfile = 'enb_' + self.testCase_id + '_s1x2log.pcap'
+				mySSH.command('echo ' + lPassWord + ' | sudo -S rm -f /tmp/' + pcapfile , '\$', 5)
+				mySSH.command('echo $USER; nohup sudo -E tshark  -i ' + eth_interface + ' -w /tmp/' + pcapfile + ' 2>&1 &','\$', 5)
+			mySSH.close()
+			
+
+
 		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
 		result = re.search('T_stdout', str(self.Initialize_eNB_args))
 		if (result is not None):
@@ -579,6 +596,10 @@ class RANManagement():
 				mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5)
 				time.sleep(5)
 		mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
+		#stopping tshark (valid if eNB and enabled in xml, will not harm otherwise)
+		logging.debug('\u001B[1m Stopping tshark \u001B[0m')
+		mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
+		time.sleep(1)
 		mySSH.close()
 		# If tracer options is on, stopping tshark on EPC side
 		result = re.search('T_stdout', str(self.Initialize_eNB_args))
@@ -597,6 +618,7 @@ class RANManagement():
 			mySSH.close()
 			logging.debug('\u001B[1m Replaying RAW record file\u001B[0m')
 			mySSH.open(lIpAddr, lUserName, lPassWord)
+			mySSH.command('killall --signal SIGKILL record', '\$', 5)
 			mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5)
 			enbLogFile = self.eNBLogFiles[int(self.eNB_instance)]
 			raw_record_file = enbLogFile.replace('.log', '_record.raw')
@@ -650,9 +672,10 @@ class RANManagement():
 		mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
 		mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5)
 		mySSH.command('cd cmake_targets', '\$', 5)
+		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*_s1x2log.pcap .','\$',20)
 		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
-		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log', '\$', 60)
-		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 5)
+		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 60)
+		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 5)
 		mySSH.close()
 
 	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML):
@@ -707,6 +730,8 @@ class RANManagement():
 		real_time_stats = {}
 		#count "problem receiving samples" msg
 		pb_receiving_samples_cnt = 0
+		#count "removing UE" msg
+		removing_ue = 0
 		#NSA specific log markers
 		nsa_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': []}
 	
@@ -903,6 +928,10 @@ class RANManagement():
 			result = re.search('\[PHY\]\s+problem receiving samples', str(line))
 			if result is not None:
 				pb_receiving_samples_cnt += 1
+			#count "Removing UE" msg
+			result = re.search('\[MAC\]\s+Removing UE', str(line))
+			if result is not None:
+				removing_ue += 1
 
 			#nsa markers logging
 			for k in nsa_markers:
@@ -992,6 +1021,11 @@ class RANManagement():
 				htmleNBFailureMsg += statMsg
 
 		else:
+			#Removing UE log
+			statMsg = '[MAC] Removing UE msg count =  '+str(removing_ue)
+			htmlMsg = statMsg+'\n'
+			logging.debug(statMsg)
+			htmleNBFailureMsg += htmlMsg
 			#nsa markers
 			statMsg = 'logfile line count = ' + str(line_cnt)			
 			htmlMsg = statMsg+'\n'

ci-scripts/xml_files/enb_ue_usrp210_band7_build.xml

@@ -32,7 +32,7 @@
 	<testCase id="010101">
 		<class>Build_eNB</class>
 		<desc>Build eNB (USRP)</desc>
-		<Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
+		<Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args>
 	</testCase>
 
 </testCaseList>

ci-scripts/xml_files/enb_usrp210_band13_build.xml

@@ -33,7 +33,7 @@
 	<testCase id="010101">
 		<class>Build_eNB</class>
 		<desc>Build eNB (USRP)</desc>
-		<Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
+		<Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args>
 	</testCase>
 
 </testCaseList>

ci-scripts/xml_files/enb_usrp210_band40_build.xml

@@ -32,7 +32,7 @@
 	<testCase id="010101">
 		<class>Build_eNB</class>
 		<desc>Build eNB (USRP)</desc>
-		<Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
+		<Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args>
 	</testCase>
 
 </testCaseList>

ci-scripts/xml_files/enb_usrp210_band7_build.xml

@@ -32,7 +32,7 @@
 	<testCase id="010101">
 		<class>Build_eNB</class>
 		<desc>Build eNB (USRP)</desc>
-		<Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
+		<Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args>
 	</testCase>
 
 </testCaseList>

ci-scripts/xml_files/fr1_nsa_quectel.xml

@@ -24,21 +24,21 @@
 	<htmlTabRef>TEST-NSA-FR1-TM1</htmlTabRef>
 	<htmlTabName>NSA Ping DL UL with QUECTEL</htmlTabName>
 	<htmlTabIcon>tasks</htmlTabIcon>
-	<repeatCount>3</repeatCount>
+	<repeatCount>1</repeatCount>
 	<TestCaseRequestedList>
  030000
  040000
  000002
  010000
+ 000001
  050000
  050001
  000001
  070000
  000001
- 070000
+ 070001
  000001
  010002
- 000001
  080001
  080000
 	</TestCaseRequestedList>
@@ -65,13 +65,14 @@
 		<eNB_instance>0</eNB_instance>
 		<eNB_serverId>0</eNB_serverId>
 		<air_interface>lte</air_interface>
+		<eNB_Trace>yes</eNB_Trace>
 	</testCase>
 
 
 	<testCase id="040000">
 		<class>Initialize_eNB</class>
 		<desc>Initialize gNB</desc>
-		<Initialize_eNB_args>-O ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpn310.conf -q</Initialize_eNB_args>
+		<Initialize_eNB_args>-O ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf -E -q</Initialize_eNB_args>
 		<eNB_instance>1</eNB_instance>
 		<eNB_serverId>1</eNB_serverId>
 		<air_interface>nr</air_interface>

ci-scripts/xml_files/if4p5_usrp210_band40_build.xml

@@ -33,7 +33,7 @@
 	<testCase id="010101">
 		<class>Build_eNB</class>
 		<desc>Build eNB (USRP -- Ethernet Fronthaul)</desc>
-		<Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
+		<Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args>
 	</testCase>
 
 </testCaseList>

ci-scripts/xml_files/if4p5_usrp210_band7_build.xml

@@ -33,7 +33,7 @@
 	<testCase id="010101">
 		<class>Build_eNB</class>
 		<desc>Build eNB (USRP -- Ethernet Fronthaul)</desc>
-		<Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
+		<Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args>
 	</testCase>
 
 </testCaseList>

ci-scripts/xml_files/x2ho_enb_usrp210_band13_build.xml

@@ -42,7 +42,7 @@
 	<testCase id="010102">
 		<class>Build_eNB</class>
 		<desc>Build Slave eNB (USRP)</desc>
-		<Build_eNB_args>-w USRP -c --eNB</Build_eNB_args>
+		<Build_eNB_args>-w USRP -c --eNB --ninja</Build_eNB_args>
 		<eNB_instance>1</eNB_instance>
 		<eNB_serverId>1</eNB_serverId>
 	</testCase>

openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c

@@ -77,7 +77,7 @@ static void nr_pdcp_entity_recv_pdu(nr_pdcp_entity_t *entity,
   }
 
   rx_deliv_sn  = entity->rx_deliv & entity->sn_max;
-  rx_deliv_hfn = (entity->rx_deliv >> entity->sn_size) & ~entity->sn_max;
+  rx_deliv_hfn = entity->rx_deliv >> entity->sn_size;
 
   if (rcvd_sn < rx_deliv_sn - entity->window_size) {
     rcvd_hfn = rx_deliv_hfn + 1;
@@ -138,7 +138,7 @@ static void nr_pdcp_entity_recv_pdu(nr_pdcp_entity_t *entity,
     entity->rx_deliv = count;
   }
 
-  if (entity->t_reordering_start != 0 && entity->rx_deliv > entity->rx_reord) {
+  if (entity->t_reordering_start != 0 && entity->rx_deliv >= entity->rx_reord) {
     /* stop and reset t-Reordering */
     entity->t_reordering_start = 0;
   }
@@ -217,6 +217,10 @@ static void check_t_reordering(nr_pdcp_entity_t *entity)
 {
   uint32_t count;
 
+  /* if t_reordering is set to "infinity" (seen as -1) then do nothing */
+  if (entity->t_reordering == -1)
+    return;
+
   if (entity->t_reordering_start == 0
       || entity->t_current <= entity->t_reordering_start + entity->t_reordering)
     return;

openair2/LAYER2/nr_pdcp/nr_pdcp_entity.h

@@ -55,8 +55,8 @@ typedef struct nr_pdcp_entity_t {
   int rb_id;
 
   int sn_size;                  /* SN size, in bits */
-  int t_reordering;             /* unit: ms */
-  int discard_timer;            /* unit: ms */
+  int t_reordering;             /* unit: ms, -1 for infinity */
+  int discard_timer;            /* unit: ms, -1 for infinity */
 
   int sn_max;                   /* (2^SN_size) - 1 */
   int window_size;              /* 2^(SN_size - 1) */

openair2/LAYER2/nr_pdcp/nr_pdcp_oai_api.c

@@ -610,11 +610,17 @@ static void add_drb_am(int is_gnb, int rnti, struct NR_DRB_ToAddMod *s,
   nr_pdcp_ue_t *ue;
 
   int drb_id = s->drb_Identity;
-  int t_reordering = decode_t_reordering(*s->pdcp_Config->t_Reordering);
+  int t_reordering;
   int sn_size_ul = decode_sn_size_ul(*s->pdcp_Config->drb->pdcp_SN_SizeUL);
   int sn_size_dl = decode_sn_size_dl(*s->pdcp_Config->drb->pdcp_SN_SizeDL);
   int discard_timer = decode_discard_timer(*s->pdcp_Config->drb->discardTimer);
 
+  /* if pdcp_Config->t_Reordering is not present, it means infinity (-1) */
+  if (s->pdcp_Config->t_Reordering != NULL)
+    t_reordering = decode_t_reordering(*s->pdcp_Config->t_Reordering);
+  else
+    t_reordering = -1;
+
   /* TODO(?): accept different UL and DL SN sizes? */
   if (sn_size_ul != sn_size_dl) {
     LOG_E(PDCP, "%s:%d:%s: fatal, bad SN sizes, must be same. ul=%d, dl=%d\n",

openair2/LAYER2/nr_pdcp/nr_pdcp_security_nea2.c

@@ -27,12 +27,21 @@
 #include <nettle/aes.h>
 #include <nettle/ctr.h>
 
+#ifndef NETTLE_VERSION_MAJOR
+/* hack: include bignum.h, not version.h because version.h does not exist
+ *       in old versions and bignum.h includes version.h (as of today).
+ *       May completely fail to work... maybe we should skip support of old
+ *       versions of nettle.
+ */
+#include <nettle/bignum.h>
+#endif
+
 void *nr_pdcp_security_nea2_init(unsigned char *ciphering_key)
 {
   void *ctx = calloc(1, nettle_aes128.context_size);
   if (ctx == NULL) exit(1);
 
-#if NETTLE_VERSION_MAJOR < 3
+#if !defined(NETTLE_VERSION_MAJOR) || NETTLE_VERSION_MAJOR < 3
   nettle_aes128.set_encrypt_key(ctx, 16, ciphering_key);
 #else
   nettle_aes128.set_encrypt_key(ctx, ciphering_key);

openair2/LAYER2/nr_rlc/nr_rlc_oai_api.c

@@ -561,7 +561,7 @@ static void max_retx_reached(void *_ue, nr_rlc_entity_t *entity)
   exit(1);
 
 rb_found:
-  LOG_D(RLC, "max RETX reached on %s %d\n",
+  LOG_E(RLC, "max RETX reached on %s %d\n",
         is_srb ? "SRB" : "DRB",
         rb_id);