ran.py 65 KB
Newer Older
Raphael Defosseux's avatar
Raphael Defosseux committed
1
#/*
Gabriele Perrone's avatar
Gabriele Perrone committed
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

31 32 33 34 35 36 37 38 39
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys              # arg
import re               # reg
import logging
import os
import time
from multiprocessing import Process, Lock, SimpleQueue
hardy's avatar
hardy committed
40
import yaml
41

42

43 44 45
#-----------------------------------------------------------
# OAI Testing modules
#-----------------------------------------------------------
Gabriele Perrone's avatar
Gabriele Perrone committed
46
import sshconnection as SSH
47 48 49 50 51 52 53
import helpreadme as HELP
import constants as CONST

#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class RANManagement():
Gabriele Perrone's avatar
Gabriele Perrone committed
54

55
	def __init__(self):
Gabriele Perrone's avatar
Gabriele Perrone committed
56 57
		
		self.prematureExit = False
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
		self.ranRepository = ''
		self.ranBranch = ''
		self.ranAllowMerge = False
		self.ranCommitID = ''
		self.ranTargetBranch = ''
		self.eNBIPAddress = ''
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.eNB1IPAddress = ''
		self.eNB1UserName = ''
		self.eNB1Password = ''
		self.eNB1SourceCodePath = ''
		self.eNB2IPAddress = ''
		self.eNB2UserName = ''
		self.eNB2Password = ''
		self.eNB2SourceCodePath = ''
		self.Build_eNB_args = ''
		self.backgroundBuild = False
		self.backgroundBuildTestId = ['', '', '']
		self.Build_eNB_forced_workspace_cleanup = False
		self.Initialize_eNB_args = ''
80
		self.imageKind = ''
81 82
		self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main
		self.eNB_instance = 0
83
		self.eNB_serverId = ['', '', '']
84 85 86
		self.eNBLogFiles = ['', '', '']
		self.eNBOptions = ['', '', '']
		self.eNBmbmsEnables = [False, False, False]
87 88 89
		self.eNBstatuses = [-1, -1, -1]
		self.flexranCtrlInstalled = False
		self.flexranCtrlStarted = False
90 91
		self.flexranCtrlDeployed = False
		self.flexranCtrlIpAddress = ''
92
		self.testCase_id = ''
93
		self.epcPcapFile = ''
94
		self.runtime_stats= ''
hardy's avatar
hardy committed
95
		self.datalog_rt_stats={}
96
		self.datalog_rt_stats_file='datalog_rt_stats.default.yaml'
97
		self.eNB_Trace = '' #if 'yes', Tshark will be launched at initialization
hardy's avatar
hardy committed
98
		self.eNB_Stats = '' #if 'yes', Statistics Monitor will be launched at initialization		
hardy's avatar
hardy committed
99
		self.USRPIPAddress = ''
Gabriele Perrone's avatar
Gabriele Perrone committed
100

hardy's avatar
hardy committed
101

102

Gabriele Perrone's avatar
Gabriele Perrone committed
103 104 105 106
#-----------------------------------------------------------
# RAN management functions
#-----------------------------------------------------------

Raphael Defosseux's avatar
Raphael Defosseux committed
107
	def BuildeNB(self, HTML):
108
		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
109
			HELP.GenericHelp(CONST.Version)
110
			sys.exit('Insufficient Parameter')
111
		if self.eNB_serverId[self.eNB_instance] == '0':
112 113 114 115
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
116
		elif self.eNB_serverId[self.eNB_instance] == '1':
117 118 119 120
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
121
		elif self.eNB_serverId[self.eNB_instance] == '2':
122 123 124 125 126
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
127
			HELP.GenericHelp(CONST.Version)
128
			sys.exit('Insufficient Parameter')
129
		logging.debug('Building on server: ' + lIpAddr)
130
		mySSH = SSH.SSHConnection()
Gabriele Perrone's avatar
Gabriele Perrone committed
131
		mySSH.open(lIpAddr, lUserName, lPassWord)
hardy's avatar
hardy committed
132 133 134
		
		# Check if we build an 5G-NR gNB or an LTE eNB or an OCP eNB
		result = re.search('--eNBocp', self.Build_eNB_args)
135
		if result is not None:
136
			self.air_interface[self.eNB_instance] = 'ocp-enb'
137 138
		else:
			result = re.search('--RU', self.Build_eNB_args)
hardy's avatar
hardy committed
139
			if result is not None:
140
				self.air_interface[self.eNB_instance] = 'oairu'
hardy's avatar
hardy committed
141
			else:
142 143 144 145 146
				result = re.search('--gNB', self.Build_eNB_args)
				if result is not None:
					self.air_interface[self.eNB_instance] = 'nr-softmodem'
				else:
					self.air_interface[self.eNB_instance] = 'lte-softmodem'
147
		
148 149 150
		# Worakround for some servers, we need to erase completely the workspace
		if self.Build_eNB_forced_workspace_cleanup:
			mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
Raphael Defosseux's avatar
Raphael Defosseux committed
151
		self.testCase_id = HTML.testCase_id
152 153 154
		# on RedHat/CentOS .git extension is mandatory
		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
		if result is not None:
155
			full_ran_repo_name = self.ranRepository.replace('git/', 'git')
156 157 158 159 160 161 162 163
		else:
			full_ran_repo_name = self.ranRepository + '.git'
		mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
		# Raphael: here add a check if git clone or git fetch went smoothly
		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
164
		mySSH.command('git advice.detachedHead false', '\$', 5)
165 166 167 168 169 170
		# Checking the BUILD INFO file
		if not self.backgroundBuild:
			mySSH.command('ls *.txt', '\$', 5)
			result = re.search('LAST_BUILD_INFO', mySSH.getBefore())
			if result is not None:
				mismatch = False
171
				mySSH.command('grep --colour=never SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
172 173 174
				result = re.search(self.ranCommitID, mySSH.getBefore())
				if result is None:
					mismatch = True
175
				mySSH.command('grep --colour=never MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
176 177 178 179
				if (self.ranAllowMerge):
					result = re.search('YES', mySSH.getBefore())
					if result is None:
						mismatch = True
180
					mySSH.command('grep --colour=never TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
181 182 183 184 185 186 187 188 189 190 191 192
					if self.ranTargetBranch == '':
						result = re.search('develop', mySSH.getBefore())
					else:
						result = re.search(self.ranTargetBranch, mySSH.getBefore())
					if result is None:
						mismatch = True
				else:
					result = re.search('NO', mySSH.getBefore())
					if result is None:
						mismatch = True
				if not mismatch:
					mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
193
					HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
194 195 196 197 198
					return

		mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
		# if the commit ID is provided use it to point to it
		if self.ranCommitID != '':
199
			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
200 201 202 203 204 205 206 207 208
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
		if (self.ranAllowMerge):
			if self.ranTargetBranch == '':
				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
209
		logging.debug(mySSH.getBefore()) # print what git said when merging/checking out
210 211 212 213 214 215 216 217
		mySSH.command('source oaienv', '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
		mySSH.command('mkdir -p log', '\$', 5)
		mySSH.command('chmod 777 log', '\$', 5)
		# no need to remove in log (git clean did the trick)
		if self.backgroundBuild:
			mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5)
			mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5)
218
			mySSH.command('echo ' + lPassWord + ' | sudo -S ls', '\$', 5)
219
			logging.debug(mySSH.getBefore()) # print current directory contents for verification
220
			mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-build.sh' + ' > ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ' + ' 2>&1 &', lUserName, 5)
221
			mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
222
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
223
			self.backgroundBuildTestId[int(self.eNB_instance)] = self.testCase_id
224 225
			return
		mySSH.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests|build have failed', 1500)
226
		mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
227
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.testCase_id, HTML)
228

Raphael Defosseux's avatar
Raphael Defosseux committed
229
	def WaitBuildeNBisFinished(self, HTML):
230
		if self.eNB_serverId[self.eNB_instance] == '0':
231 232 233 234
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
235
		elif self.eNB_serverId[self.eNB_instance] == '1':
236 237 238 239
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
240
		elif self.eNB_serverId[self.eNB_instance] == '2':
241 242 243 244 245
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
246
			HELP.GenericHelp(CONST.Version)
247
			sys.exit('Insufficient Parameter')
248
		logging.debug('Waiting for end of build on server: ' + lIpAddr)
Gabriele Perrone's avatar
Gabriele Perrone committed
249
		mySSH = SSH.SSHConnection()
250 251 252 253
		mySSH.open(lIpAddr, lUserName, lPassWord)
		count = 40
		buildOAIprocess = True
		while (count > 0) and buildOAIprocess:
254
			mySSH.command('ps aux | grep --color=never build_ | grep -v grep', '\$', 6)
255 256 257 258 259 260
			result = re.search('build_oai', mySSH.getBefore())
			if result is None:
				buildOAIprocess = False
			else:
				count -= 1
				time.sleep(30)
261
		mySSH.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
262
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.backgroundBuildTestId[int(self.eNB_instance)], HTML)
263

Raphael Defosseux's avatar
Raphael Defosseux committed
264 265
	def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId, HTML):
		HTML.testCase_id=testcaseId
hardy's avatar
hardy committed
266

267
		mySSH = SSH.SSHConnection()
268
		mySSH.open(lIpAddr, lUserName, lPassWord)
Gabriele Perrone's avatar
Gabriele Perrone committed
269
		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 3)
270 271
		mySSH.command('ls ran_build/build', '\$', 3)
		mySSH.command('ls ran_build/build', '\$', 3)
hardy's avatar
hardy committed
272 273

		#check if we have the build corresponding to the air interface keywords (nr-softmode, lte-softmodem, ocp-enb)
274
		logging.info('CHECK Build with IP='+lIpAddr+' SourcePath='+lSourcePath)
275
		result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore())
276
		if result is None:
hardy's avatar
hardy committed
277
			buildStatus = False #if not, build failed
278
		else:
hardy's avatar
hardy committed
279
			buildStatus = True 
280 281 282 283 284 285 286 287 288 289 290
			# Generating a BUILD INFO file
			mySSH.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2)
			mySSH.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			if (self.ranAllowMerge):
				mySSH.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				if self.ranTargetBranch == '':
					mySSH.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				else:
					mySSH.command('echo "TGT_BRANCH: ' + self.ranTargetBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			else:
				mySSH.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
hardy's avatar
hardy committed
291 292
				
				
293 294 295
		mySSH.command('mkdir -p build_log_' + testcaseId, '\$', 5)
		mySSH.command('mv log/* ' + 'build_log_' + testcaseId, '\$', 5)
		mySSH.command('mv compile_oai_enb.log ' + 'build_log_' + testcaseId, '\$', 5)
296
		if self.eNB_serverId[self.eNB_instance] != '0':
297 298 299 300 301 302 303 304 305 306 307 308
			mySSH.command('cd cmake_targets', '\$', 5)
			mySSH.command('if [ -e tmp_build' + testcaseId + '.zip ]; then rm -f tmp_build' + testcaseId + '.zip; fi', '\$', 5)
			mySSH.command('zip -r -qq tmp_build' + testcaseId + '.zip build_log_' + testcaseId, '\$', 5)
			mySSH.close()
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				os.remove('./tmp_build' + testcaseId + '.zip')
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/tmp_build' + testcaseId + '.zip', '.')
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './tmp_build' + testcaseId + '.zip', self.eNBSourceCodePath + '/cmake_targets/.')
				os.remove('./tmp_build' + testcaseId + '.zip')
				mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
				mySSH.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5)
Remi Hardy's avatar
Remi Hardy committed
309
				#-qq quiet / -u update orcreate files
310
				mySSH.command('unzip -o -u -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5)
311 312 313 314 315
				mySSH.command('rm -f tmp_build' + testcaseId + '.zip', '\$', 5)
				mySSH.close()
		else:
			mySSH.close()

hardy's avatar
hardy committed
316
		#generate logging info depending on buildStatus and air interface
317
		if buildStatus:
318
			logging.info('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Pass\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
319
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
320
		else:
321
			logging.error('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Failed\u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
322 323
			HTML.CreateHtmlTestRow(self.Build_eNB_args, 'KO', CONST.ALL_PROCESSES_OK)
			HTML.CreateHtmlTabFooter(False)
324 325
			sys.exit(1)

Raphael Defosseux's avatar
Raphael Defosseux committed
326
	def InitializeeNB(self, HTML, EPC):
327
		if self.eNB_serverId[self.eNB_instance] == '0':
328 329 330 331
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
332
		elif self.eNB_serverId[self.eNB_instance] == '1':
333 334 335 336
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
337
		elif self.eNB_serverId[self.eNB_instance] == '2':
338 339 340 341 342
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
343
			HELP.GenericHelp(CONST.Version)
344
			sys.exit('Insufficient Parameter')
345
		logging.debug('Starting eNB/gNB on server: ' + lIpAddr)
346

Raphael Defosseux's avatar
Raphael Defosseux committed
347
		self.testCase_id = HTML.testCase_id
Gabriele Perrone's avatar
Gabriele Perrone committed
348
		mySSH = SSH.SSHConnection()
hardy's avatar
hardy committed
349
		cwd = os.getcwd()
hardy's avatar
hardy committed
350
		mySSH.copyout(lIpAddr,lUserName,lPassWord, cwd + "/active_net_interfaces.awk", "/tmp")
Gabriele Perrone's avatar
Gabriele Perrone committed
351
		
hardy's avatar
hardy committed
352 353
		#reboot USRP if requested in xml
		if self.USRPIPAddress!='':
354
			logging.debug('USRP '+ self.USRPIPAddress +': reboot request')
hardy's avatar
hardy committed
355 356 357 358 359 360 361 362
			mySSH.open(lIpAddr, lUserName, lPassWord)
			cmd2usrp='ssh root@'+self.USRPIPAddress+' reboot'
			mySSH.command2(cmd2usrp,1)
			mySSH.close()
			logging.debug('Waiting for USRP to be ready')
			time.sleep(120)


Gabriele Perrone's avatar
Gabriele Perrone committed
363
		if (self.pStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
364 365
			HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus)
			HTML.CreateHtmlTabFooter(False)
366
			sys.exit(1)
hardy's avatar
hardy committed
367 368 369 370 371 372 373

		#Get pcap on enb and/or gnb if enabled in the xml 
		if self.eNB_Trace=='yes':
			if ((self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb')):
				pcapfile_prefix="enb_"
			else:
				pcapfile_prefix="gnb_"
374
			mySSH.open(lIpAddr, lUserName, lPassWord)
Laurent Thomas's avatar
Laurent Thomas committed
375 376
			eth_interface = 'any'
			fltr = 'sctp'
377
			logging.debug('\u001B[1m Launching tshark on xNB on interface ' + eth_interface + ' with filter "' + fltr + '"\u001B[0m')
Laurent Thomas's avatar
Laurent Thomas committed
378 379 380
			pcapfile = pcapfile_prefix + self.testCase_id + '_log.pcap'
			mySSH.command('echo ' + lPassWord + ' | sudo -S rm -f /tmp/' + pcapfile , '\$', 5)
			mySSH.command('echo $USER; nohup sudo -E tshark  -i ' + eth_interface + ' -f "' + fltr + '" -w /tmp/' + pcapfile + ' > /dev/null 2>&1 &','\$', 5)
381 382 383 384
			mySSH.close()
			


385
		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
386 387 388 389 390 391 392
		if EPC.IPAddress != "none":
			localEpcIpAddr = EPC.IPAddress
			localEpcUserName = EPC.UserName
			localEpcPassword = EPC.Password
			mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
			eth_interface = 'any'
			fltr = 'sctp'
393
			logging.debug('\u001B[1m Launching tshark on EPC on interface ' + eth_interface + ' with filter "' + fltr + '"\u001B[0m')
394 395 396 397 398
			self.epcPcapFile = 'enb_' + self.testCase_id + '_s1log.pcap'
			mySSH.command('echo ' + localEpcPassword + ' | sudo -S rm -f /tmp/' + self.epcPcapFile , '\$', 5)
			mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -f "' + fltr + '" -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5)
			mySSH.close()

399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		# Initialize_eNB_args usually start with -O and followed by the location in repository
		full_config_file = self.Initialize_eNB_args.replace('-O ','')
		extra_options = ''
		extIdx = full_config_file.find('.conf')
		if (extIdx > 0):
			extra_options = full_config_file[extIdx + 5:]
			# if tracer options is on, compiling and running T Tracer
			result = re.search('T_stdout', str(extra_options))
			if result is not None:
				logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
				mySSH.command('cd common/utils/T/tracer', '\$', 5)
				mySSH.command('make', '\$', 10)
				mySSH.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', lUserName, 5)
				mySSH.command('cd ' + lSourcePath, '\$', 5)
			full_config_file = full_config_file[:extIdx + 5]
			config_path, config_file = os.path.split(full_config_file)
		else:
			sys.exit('Insufficient Parameter')
		ci_full_config_file = config_path + '/ci-' + config_file
		rruCheck = False
		result = re.search('^rru|^rcc|^du.band', str(config_file))
		if result is not None:
			rruCheck = True
		# do not reset board twice in IF4.5 case
		result = re.search('^rru|^enb|^du.band', str(config_file))
		if result is not None:
427
			mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 180)
428 429 430 431 432
			result = re.search('type: b200', mySSH.getBefore())
			if result is not None:
				logging.debug('Found a B2xx device --> resetting it')
				mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
				# Reloading FGPA bin firmware
433
				mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 180)
434 435
		# Make a copy and adapt to EPC / eNB IP addresses
		mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
436 437
		localMmeIpAddr = EPC.MmeIPAddress
		mySSH.command('sed -i -e \'s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
438
		mySSH.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
439
		mySSH.command('sed -i -e \'s/CI_GNB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
440 441 442
		mySSH.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RRU1_IP_ADDR/' + self.eNB1IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RRU2_IP_ADDR/' + self.eNB2IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
443
		mySSH.command('sed -i -e \'s/CI_FR1_CTL_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
444
		if (self.flexranCtrlInstalled and self.flexranCtrlStarted) or self.flexranCtrlDeployed:
445
			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "yes";/\' ' + ci_full_config_file, '\$', 2);
446
			mySSH.command('sed -i -e \'s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/\' ' + ci_full_config_file, '\$', 2);
447 448 449
		else:
			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "no";/\' ' + ci_full_config_file, '\$', 2);
		self.eNBmbmsEnables[int(self.eNB_instance)] = False
450
		mySSH.command('grep --colour=never enable_enb_m2 ' + ci_full_config_file, '\$', 2);
451 452 453 454 455 456 457 458 459 460 461 462
		result = re.search('yes', mySSH.getBefore())
		if result is not None:
			self.eNBmbmsEnables[int(self.eNB_instance)] = True
			logging.debug('\u001B[1m MBMS is enabled on this eNB\u001B[0m')
		result = re.search('noS1', str(self.Initialize_eNB_args))
		eNBinNoS1 = False
		if result is not None:
			eNBinNoS1 = True
			logging.debug('\u001B[1m eNB is in noS1 configuration \u001B[0m')
		# Launch eNB with the modified config file
		mySSH.command('source oaienv', '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
463
		if self.air_interface[self.eNB_instance] == 'nr-softmodem':
464
			mySSH.command('if [ -e rbconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm rbconfig.raw; fi', '\$', 5)
465
			mySSH.command('if [ -e reconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm reconfig.raw; fi', '\$', 5)
466
		# NOTE: WE SHALL do a check if the executable is present (in case build went wrong)
467 468 469 470 471 472 473 474 475 476 477

		#hack UHD_RFNOC_DIR variable for gNB / N310 on RHEL8 server:
		#if the USRP address is in the xml then we are using an eth USRP (N3xx)
		if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
			gNB = False
		else:
			gNB = True
		if ((self.USRPIPAddress!='') and (gNB==True)):
			mySSH.command('echo ' + lPassWord + ' | echo "ulimit -c unlimited && sudo UHD_RFNOC_DIR=/usr/local/share/uhd/rfnoc ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		#otherwise the regular command is ok
		else:
478
			mySSH.command('echo "ulimit -c unlimited && catchsegv ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
479

480 481
		mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)
482
		mySSH.command('echo $USER; nohup sudo -E stdbuf -o0 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10)
483 484


485
		#stats monitoring during runtime
486
		time.sleep(20)
487 488
		monitor_file='../ci-scripts/stats_monitor.py'
		conf_file='../ci-scripts/stats_monitor_conf.yaml'
489
		if self.eNB_Stats=='yes':
490
			if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
491
				mySSH.command('echo $USER; nohup python3 ' + monitor_file + ' ' + conf_file + ' ' + self.testCase_id + ' enb 2>&1 > enb_stats_monitor_execution.log &', '\$', 5)
492
			else:
493
				mySSH.command('echo $USER; nohup python3 ' + monitor_file + ' ' + conf_file + ' ' + self.testCase_id + ' gnb 2>&1 > gnb_stats_monitor_execution.log &', '\$', 5)
494 495 496



497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512
		self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log'
		if extra_options != '':
			self.eNBOptions[int(self.eNB_instance)] = extra_options
		time.sleep(6)
		doLoop = True
		loopCounter = 20
		enbDidSync = False
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
				# In case of T tracer recording, we may need to kill it
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					mySSH.command('killall --signal SIGKILL record', '\$', 5)
				mySSH.close()
				doLoop = False
513
				logging.error('\u001B[1;37;41m eNB/gNB/ocp-eNB logging system did not show got sync! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
514
				HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'KO', CONST.ALL_PROCESSES_OK)
515
				# In case of T tracer recording, we need to kill tshark on EPC side
Laurent Thomas's avatar
Laurent Thomas committed
516 517 518 519
				localEpcIpAddr = EPC.IPAddress
				localEpcUserName = EPC.UserName
				localEpcPassword = EPC.Password
				mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
520
				logging.debug('\u001B[1m Stopping tshark on EPC \u001B[0m')
Laurent Thomas's avatar
Laurent Thomas committed
521 522 523 524 525 526 527 528
				mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
				if self.epcPcapFile  != '':
					mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
				mySSH.close()
				if self.epcPcapFile != '':
					copyin_res = mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
					if (copyin_res == 0):
						mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
529 530 531
				self.prematureExit = True
				return
			else:
532
				mySSH.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting|Started"', '\$', 4)
533 534 535 536 537 538 539 540 541 542 543
				if rruCheck:
					result = re.search('wait RUs', mySSH.getBefore())
				else:
					result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore())
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
					enbDidSync = True
					time.sleep(10)

544 545 546 547 548
		rruCheck = False
		result = re.search('^rru|^du.band', str(config_file))
		if result is not None:
			rruCheck = True
		if enbDidSync and eNBinNoS1 and not rruCheck:
549 550 551 552 553 554 555 556 557 558 559 560 561 562 563
			mySSH.command('ifconfig oaitun_enb1', '\$', 4)
			mySSH.command('ifconfig oaitun_enb1', '\$', 4)
			result = re.search('inet addr:1|inet 1', mySSH.getBefore())
			if result is not None:
				logging.debug('\u001B[1m oaitun_enb1 interface is mounted and configured\u001B[0m')
			else:
				logging.error('\u001B[1m oaitun_enb1 interface is either NOT mounted or NOT configured\u001B[0m')
			if self.eNBmbmsEnables[int(self.eNB_instance)]:
				mySSH.command('ifconfig oaitun_enm1', '\$', 4)
				result = re.search('inet addr', mySSH.getBefore())
				if result is not None:
					logging.debug('\u001B[1m oaitun_enm1 interface is mounted and configured\u001B[0m')
				else:
					logging.error('\u001B[1m oaitun_enm1 interface is either NOT mounted or NOT configured\u001B[0m')
		if enbDidSync:
564
			self.eNBstatuses[int(self.eNB_instance)] = int(self.eNB_serverId[self.eNB_instance])
565 566

		mySSH.close()
hardy's avatar
hardy committed
567 568


Raphael Defosseux's avatar
Raphael Defosseux committed
569
		HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'OK', CONST.ALL_PROCESSES_OK)
570
		logging.debug('\u001B[1m Initialize eNB/gNB/ocp-eNB Completed\u001B[0m')
571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590

	def CheckeNBProcess(self, status_queue):
		try:
			# At least the instance 0 SHALL be on!
			if self.eNBstatuses[0] == 0:
				lIpAddr = self.eNBIPAddress
				lUserName = self.eNBUserName
				lPassWord = self.eNBPassword
			elif self.eNBstatuses[0] == 1:
				lIpAddr = self.eNB1IPAddress
				lUserName = self.eNB1UserName
				lPassWord = self.eNB1Password
			elif self.eNBstatuses[0] == 2:
				lIpAddr = self.eNB2IPAddress
				lUserName = self.eNB2UserName
				lPassWord = self.eNB2Password
			else:
				lIpAddr = self.eNBIPAddress
				lUserName = self.eNBUserName
				lPassWord = self.eNBPassword
Gabriele Perrone's avatar
Gabriele Perrone committed
591
			mySSH = SSH.SSHConnection()
592
			mySSH.open(lIpAddr, lUserName, lPassWord)
593 594 595 596 597 598
			if self.air_interface[self.eNB_instance] == '':
				pattern = 'softmodem'
			else:
				pattern = self.air_interface[self.eNB_instance]
			mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + pattern + ' | grep -v grep', '\$', 5)
			result = re.search(pattern, mySSH.getBefore())
599 600
			if result is None:
				logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
Gabriele Perrone's avatar
Gabriele Perrone committed
601
				status_queue.put(CONST.ENB_PROCESS_FAILED)
602
			else:
Gabriele Perrone's avatar
Gabriele Perrone committed
603
				status_queue.put(CONST.ENB_PROCESS_OK)
604 605 606 607
			mySSH.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

Raphael Defosseux's avatar
Raphael Defosseux committed
608
	def TerminateeNB(self, HTML, EPC):
609
		if self.eNB_serverId[self.eNB_instance] == '0':
610 611 612 613
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
614
		elif self.eNB_serverId[self.eNB_instance] == '1':
615 616 617 618
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
619
		elif self.eNB_serverId[self.eNB_instance] == '2':
620 621 622 623 624
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
625 626
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
627
		logging.debug('Stopping eNB/gNB on server: ' + lIpAddr)
Gabriele Perrone's avatar
Gabriele Perrone committed
628
		mySSH = SSH.SSHConnection()
629 630
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
631
		if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
632 633 634
			nodeB_prefix = 'e'
		else:
			nodeB_prefix = 'g'
hardy's avatar
hardy committed
635 636
		mySSH.command('stdbuf -o0  ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5)
		result = re.search('(-softmodem|ocp)', mySSH.getBefore())
637
		if result is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
638
			mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT -r .*-softmodem ocp-enb || true', '\$', 5)
639
			time.sleep(10)
hardy's avatar
hardy committed
640 641
			mySSH.command('stdbuf -o0  ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5)
			result = re.search('(-softmodem|ocp)', mySSH.getBefore())
642
			if result is not None:
Raphael Defosseux's avatar
Raphael Defosseux committed
643
				mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5)
644 645
				time.sleep(5)
		mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
646
		#stopping tshark (valid if eNB and enabled in xml, will not harm otherwise)
647
		logging.debug('\u001B[1m Stopping tshark on xNB \u001B[0m')
648 649
		mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
		time.sleep(1)
650
		mySSH.close()
651
		if EPC.IPAddress != "none" and EPC.IPAddress != '':
Raphael Defosseux's avatar
Raphael Defosseux committed
652 653 654
			localEpcIpAddr = EPC.IPAddress
			localEpcUserName = EPC.UserName
			localEpcPassword = EPC.Password
655
			logging.debug('\u001B[1m Stopping tshark on EPC (' + localEpcIpAddr + ') \u001B[0m')
656 657
			mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
			mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
658
			time.sleep(1)
659 660 661 662
			if self.epcPcapFile != '':
				mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
				mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
				mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
663
			mySSH.command('killall --signal SIGKILL record', '\$', 5)
664
			mySSH.close()
665 666 667 668
		# if T tracer was run with option 0 (no logs), analyze logs
		# from textlog, otherwise do normal analysis (e.g., option 2)
		result = re.search('T_stdout 0', str(self.Initialize_eNB_args))
		if (result is not None):
669 670 671
			logging.debug('\u001B[1m Replaying RAW record file\u001B[0m')
			mySSH.open(lIpAddr, lUserName, lPassWord)
			mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5)
672 673 674
			enbLogFile = self.eNBLogFiles[int(self.eNB_instance)]
			raw_record_file = enbLogFile.replace('.log', '_record.raw')
			replay_log_file = enbLogFile.replace('.log', '_replay.log')
675 676 677 678 679 680 681 682
			extracted_txt_file = enbLogFile.replace('.log', '_extracted_messages.txt')
			extracted_log_file = enbLogFile.replace('.log', '_extracted_messages.log')
			mySSH.command('./extract_config -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + extracted_txt_file, '\$', 5)
			mySSH.command('echo $USER; nohup ./replay -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', lUserName, 5)
			mySSH.command('./textlog -d ' +  lSourcePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + lSourcePath + '/cmake_targets/' + extracted_log_file, '\$', 5)
			mySSH.close()
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + extracted_log_file, '.')
			logging.debug('\u001B[1m Analyzing eNB replay logfile \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
683 684
			logStatus = self.AnalyzeLogFile_eNB(extracted_log_file, HTML)
			HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
685 686 687 688 689 690 691 692
			self.eNBLogFiles[int(self.eNB_instance)] = ''
		else:
			analyzeFile = False
			if self.eNBLogFiles[int(self.eNB_instance)] != '':
				analyzeFile = True
				fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
				self.eNBLogFiles[int(self.eNB_instance)] = ''
			if analyzeFile:
693 694 695 696 697
				#*stats.log files + pickle + png
				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*stats.log', '.')
				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.pickle', '.')
				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.png', '.')
				#
698 699 700
				copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
				if (copyin_res == -1):
					logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
701 702
					HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
					HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
703 704
					self.eNBmbmsEnables[int(self.eNB_instance)] = False
					return
705
				if self.eNB_serverId[self.eNB_instance] != '0':
706
					#*stats.log files + pickle + png
hardy's avatar
hardy committed
707 708

					#debug / tentative
hardy's avatar
hardy committed
709 710
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrL1_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrMAC_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
hardy's avatar
hardy committed
711
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './gnb_stats_monitor.pickle', self.eNBSourceCodePath + '/cmake_targets/')
712
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './gnb_stats_monitor.png', self.eNBSourceCodePath + '/cmake_targets/')
713
					#
714 715
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze)
Raphael Defosseux's avatar
Raphael Defosseux committed
716
				logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML)
717
				if (logStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
718
					HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
719 720 721
					#display rt stats for gNB only
					if len(self.datalog_rt_stats)!=0 and nodeB_prefix == 'g':
						HTML.CreateHtmlDataLogTable(self.datalog_rt_stats)
hardy's avatar
hardy committed
722
					self.prematureExit = True
723 724 725
					self.eNBmbmsEnables[int(self.eNB_instance)] = False
					return
				else:
Raphael Defosseux's avatar
Raphael Defosseux committed
726
					HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
727
			else:
Raphael Defosseux's avatar
Raphael Defosseux committed
728
				HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
729 730
		#display rt stats for gNB only
		if len(self.datalog_rt_stats)!=0 and nodeB_prefix == 'g':
hardy's avatar
hardy committed
731
			HTML.CreateHtmlDataLogTable(self.datalog_rt_stats)
732 733 734 735
		self.eNBmbmsEnables[int(self.eNB_instance)] = False
		self.eNBstatuses[int(self.eNB_instance)] = -1

	def LogCollecteNB(self):
Gabriele Perrone's avatar
Gabriele Perrone committed
736
		mySSH = SSH.SSHConnection()
737 738 739
		mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
hardy's avatar
hardy committed
740 741
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20)
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20)
742
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
743
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log* iperf*.log log/*/*.log log/*/*.pcap', '\$', 60)
744 745 746
		result = re.search('core.\d+', mySSH.getBefore())
		if result is not None:
			mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip core* ran_build/build/{lte,nr}-softmodem', '\$', 60) # add core and executable to zip
747
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log* iperf*.log log/*/*.log log/*/*.pcap', '\$', 15)
748
		mySSH.close()
Gabriele Perrone's avatar
Gabriele Perrone committed
749

750
	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}):
Gabriele Perrone's avatar
Gabriele Perrone committed
751 752 753 754 755 756 757 758 759
		if (not os.path.isfile('./' + eNBlogFile)):
			return -1
		enb_log_file = open('./' + eNBlogFile, 'r')
		exitSignalReceived = False
		foundAssertion = False
		msgAssertion = ''
		msgLine = 0
		foundSegFault = False
		foundRealTimeIssue = False
760
		foundRealTimeIssue_cnt = 0
Gabriele Perrone's avatar
Gabriele Perrone committed
761 762 763 764 765 766 767 768 769 770 771 772
		rrcSetupComplete = 0
		rrcReleaseRequest = 0
		rrcReconfigRequest = 0
		rrcReconfigComplete = 0
		rrcReestablishRequest = 0
		rrcReestablishComplete = 0
		rrcReestablishReject = 0
		rlcDiscardBuffer = 0
		rachCanceledProcedure = 0
		uciStatMsgCount = 0
		pdcpFailure = 0
		ulschFailure = 0
773 774
		ulschAllocateCCEerror = 0
		uplinkSegmentsAborted = 0
775
		ulschReceiveOK = 0
776
		gnbRxTxWakeUpFailure = 0
777
		gnbTxWriteThreadEnabled = False
Gabriele Perrone's avatar
Gabriele Perrone committed
778 779 780
		cdrxActivationMessageCount = 0
		dropNotEnoughRBs = 0
		mbmsRequestMsg = 0
781
		htmleNBFailureMsg = ''
Gabriele Perrone's avatar
Gabriele Perrone committed
782 783 784
		isRRU = False
		isSlave = False
		slaveReceivesFrameResyncCmd = False
Gabriele Perrone's avatar
Gabriele Perrone committed
785
		X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
786 787
		X2HO_inNbProcedures = 0
		X2HO_outNbProcedures = 0
788
		global_status = CONST.ALL_PROCESSES_OK
789 790 791 792 793 794
		# Runtime statistics
		runTime = ''
		userTime = ''
		systemTime = ''
		maxPhyMemUsage = ''
		nbContextSwitches = ''
hardy's avatar
hardy committed
795
		#NSA FR1 check
hardy's avatar
hardy committed
796
		NSA_RAPROC_PUSCH_check = 0
hardy's avatar
hardy committed
797 798
		#dlsch and ulsch statistics (dictionary)
		dlsch_ulsch_stats = {}
799
		#real time statistics (dictionary)
800
		real_time_stats = {}
Remi Hardy's avatar
Remi Hardy committed
801 802
		#count "problem receiving samples" msg
		pb_receiving_samples_cnt = 0
803 804
		#count "removing UE" msg
		removing_ue = 0
hardy's avatar
hardy committed
805 806
		#count"X2AP-PDU"
		x2ap_pdu = 0
hardy's avatar
hardy committed
807 808
		#gnb specific log markers
		gnb_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': [], 'Detected UL Failure on PUSCH':[]}
809
		nodeB_prefix_found = False
810
		RealTimeProcessingIssue = False
811 812
		DLRetxIssue = False
		ULRetxIssue = False
813 814
		nrRrcRcfgComplete = 0
		harqFeedbackPast = 0
815
		showedByeMsg = False # last line is Bye. -> stopped properly
816
	
817
		line_cnt=0 #log file line counter
Gabriele Perrone's avatar
Gabriele Perrone committed
818
		for line in enb_log_file.readlines():
819
			line_cnt+=1
820 821 822 823 824 825 826 827 828 829
			# Detection of eNB/gNB from a container log
			result = re.search('Starting eNB soft modem', str(line))
			if result is not None:
				nodeB_prefix_found = True
				nodeB_prefix = 'e'
			result = re.search('Starting gNB soft modem', str(line))
			if result is not None:
				nodeB_prefix_found = True
				nodeB_prefix = 'g'
			result = re.search('Run time:' ,str(line))
830 831 832 833 834 835 836
			# Runtime statistics
			result = re.search('Run time:' ,str(line))
			if result is not None:
				runTime = str(line).strip()
			if runTime != '':
				result = re.search('Time executing user inst', str(line))
				if result is not None:
837 838
					fields=line.split(':')
					userTime = 'userTime : ' + fields[1].replace('\n','')
839 840
				result = re.search('Time executing system inst', str(line))
				if result is not None:
841 842
					fields=line.split(':')
					systemTime = 'systemTime : ' + fields[1].replace('\n','')
843 844
				result = re.search('Max. Phy. memory usage:', str(line))
				if result is not None:
845 846
					fields=line.split(':')
					maxPhyMemUsage = 'maxPhyMemUsage : ' + fields[1].replace('\n','')
847 848
				result = re.search('Number of context switch.*process origin', str(line))
				if result is not None:
849 850
					fields=line.split(':')
					nbContextSwitches = 'nbContextSwitches : ' + fields[1].replace('\n','')
Gabriele Perrone's avatar
Gabriele Perrone committed
851
			if X2HO_state == CONST.X2_HO_REQ_STATE__IDLE:
Gabriele Perrone's avatar
Gabriele Perrone committed
852 853
				result = re.search('target eNB Receives X2 HO Req X2AP_HANDOVER_REQ', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
854
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ
Gabriele Perrone's avatar
Gabriele Perrone committed
855 856
				result = re.search('source eNB receives the X2 HO ACK X2AP_HANDOVER_REQ_ACK', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
857 858
					X2HO_state = CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ:
Gabriele Perrone's avatar
Gabriele Perrone committed
859 860
				result = re.search('Received LTE_RRCConnectionReconfigurationComplete from UE', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
861 862
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE:
Gabriele Perrone's avatar
Gabriele Perrone committed
863 864
				result = re.search('issue rrc_eNB_send_PATH_SWITCH_REQ', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
865 866
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ:
Gabriele Perrone's avatar
Gabriele Perrone committed
867 868
				result = re.search('received path switch ack S1AP_PATH_SWITCH_REQ_ACK', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
869
					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
870
					X2HO_inNbProcedures += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
871
			if X2HO_state == CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK:
Gabriele Perrone's avatar
Gabriele Perrone committed
872 873
				result = re.search('source eNB receives the X2 UE CONTEXT RELEASE X2AP_UE_CONTEXT_RELEASE', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
874
					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
875 876 877 878 879 880 881 882 883
					X2HO_outNbProcedures += 1

			if self.eNBOptions[int(self.eNB_instance)] != '':
				res1 = re.search('max_rxgain (?P<requested_option>[0-9]+)', self.eNBOptions[int(self.eNB_instance)])
				res2 = re.search('max_rxgain (?P<applied_option>[0-9]+)',  str(line))
				if res1 is not None and res2 is not None:
					requested_option = int(res1.group('requested_option'))
					applied_option = int(res2.group('applied_option'))
					if requested_option == applied_option:
884
						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ok-circle"></span> Command line option(s) correctly applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
885
					else:
886
						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ban-circle"></span> Command line option(s) NOT applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901
			result = re.search('Exiting OAI softmodem', str(line))
			if result is not None:
				exitSignalReceived = True
			result = re.search('[Ss]egmentation [Ff]ault', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Cc]ore [dD]ump', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Aa]ssertion', str(line))
			if result is not None and not exitSignalReceived:
				foundAssertion = True
			result = re.search('LLL', str(line))
			if result is not None and not exitSignalReceived:
				foundRealTimeIssue = True
902
				foundRealTimeIssue_cnt += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
			if foundAssertion and (msgLine < 3):
				msgLine += 1
				msgAssertion += str(line)
			result = re.search('Setting function for RU', str(line))
			if result is not None:
				isRRU = True
			if isRRU:
				result = re.search('RU 0 is_slave=yes', str(line))
				if result is not None:
					isSlave = True
				if isSlave:
					result = re.search('Received RRU_frame_resynch command', str(line))
					if result is not None:
						slaveReceivesFrameResyncCmd = True
			result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line))
			if result is not None:
				rrcSetupComplete += 1
			result = re.search('Generate LTE_RRCConnectionRelease|Generate RRCConnectionRelease', str(line))
hardy's avatar
hardy committed
921
			if result is not None:				rrcReleaseRequest += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945
			result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line))
			if result is not None:
				rrcReconfigRequest += 1
			result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line))
			if result is not None:
				rrcReconfigComplete += 1
			result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line))
			if result is not None:
				rrcReestablishRequest += 1
			result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line))
			if result is not None:
				rrcReestablishComplete += 1
			result = re.search('LTE_RRCConnectionReestablishmentReject', str(line))
			if result is not None:
				rrcReestablishReject += 1
			result = re.search('CDRX configuration activated after RRC Connection', str(line))
			if result is not None:
				cdrxActivationMessageCount += 1
			result = re.search('uci->stat', str(line))
			if result is not None:
				uciStatMsgCount += 1
			result = re.search('PDCP.*Out of Resources.*reason', str(line))
			if result is not None:
				pdcpFailure += 1
946 947 948
			result = re.search('could not wakeup gNB rxtx process', str(line))
			if result is not None:
				gnbRxTxWakeUpFailure += 1
949 950 951
			result = re.search('tx write thread ready', str(line))
			if result is not None:
				gnbTxWriteThreadEnabled = True
952
			result = re.search('ULSCH in error in round|ULSCH 0 in error', str(line))
Gabriele Perrone's avatar
Gabriele Perrone committed
953 954
			if result is not None:
				ulschFailure += 1
955 956 957 958 959 960
			result = re.search('ERROR ALLOCATING CCEs', str(line))
			if result is not None:
				ulschAllocateCCEerror += 1
			result = re.search('uplink segment error.*aborted [1-9] segments', str(line))
			if result is not None:
				uplinkSegmentsAborted += 1
961 962 963
			result = re.search('ULSCH received ok', str(line))
			if result is not None:
				ulschReceiveOK += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
964 965 966 967 968 969 970 971 972 973 974 975 976
			result = re.search('BAD all_segments_received', str(line))
			if result is not None:
				rlcDiscardBuffer += 1
			result = re.search('Canceled RA procedure for UE rnti', str(line))
			if result is not None:
				rachCanceledProcedure += 1
			result = re.search('dropping, not enough RBs', str(line))
			if result is not None:
				dropNotEnoughRBs += 1
			if self.eNBmbmsEnables[int(self.eNB_instance)]:
				result = re.search('MBMS USER-PLANE.*Requesting.*bytes from RLC', str(line))
				if result is not None:
					mbmsRequestMsg += 1
hardy's avatar
hardy committed
977
			#FR1 NSA test : add new markers to make sure gNB is used
francescomani's avatar
francescomani committed
978
			result = re.search('\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI 0x[0-9a-fA-F]+ received correctly, adding UE MAC Context RNTI 0x[0-9a-fA-F]+', str(line))
hardy's avatar
hardy committed
979 980
			if result is not None:
				NSA_RAPROC_PUSCH_check = 1
hardy's avatar
hardy committed
981
			#dlsch and ulsch statistics
982
			#keys below are the markers we are loooking for, loop over this keys list
hardy's avatar
hardy committed
983
			#everytime these markers are found in the log file, the previous ones are overwritten in the dict
Remi Hardy's avatar
Remi Hardy committed
984
			#eventually we record and print only the last occurence 
hardy's avatar
hardy committed
985
			keys = {'UE ID','dlsch_rounds','dlsch_total_bytes','ulsch_rounds','ulsch_total_bytes_scheduled'}
hardy's avatar
hardy committed
986
			for k in keys:
987
				result = re.search(k, line)
hardy's avatar
hardy committed
988
				if result is not None:
989 990 991 992 993 994 995 996 997 998
					ue_prefix = 'ue0'
					ue_res = re.search('UE ID 1|UE 1:', line)
					if ue_res is not None:
						ue_prefix = 'ue1'
					ue_res = re.search('UE ID 2|UE 2:', line)
					if ue_res is not None:
						ue_prefix = 'ue2'
					ue_res = re.search('UE ID 3|UE 3:', line)
					if ue_res is not None:
						ue_prefix = 'ue3'
Remi Hardy's avatar
Remi Hardy committed
999
					#remove 1- all useless char before relevant info (ulsch or dlsch) 2- trailing char
1000
					dlsch_ulsch_stats[ue_prefix+k]=re.sub(r'^.*\]\s+', r'' , line.rstrip())
1001

1002 1003 1004 1005 1006 1007 1008
			result = re.search('Received NR_RRCReconfigurationComplete from UE', str(line))
			if result is not None:
				nrRrcRcfgComplete += 1
			result = re.search('HARQ feedback is in the past', str(line))
			if result is not None:
				harqFeedbackPast += 1

hardy's avatar
hardy committed
1009

Remi Hardy's avatar
Remi Hardy committed
1010 1011 1012
			#count "problem receiving samples" msg
			result = re.search('\[PHY\]\s+problem receiving samples', str(line))
			if result is not None:
1013
				pb_receiving_samples_cnt += 1
1014 1015 1016 1017
			#count "Removing UE" msg
			result = re.search('\[MAC\]\s+Removing UE', str(line))
			if result is not None:
				removing_ue += 1
hardy's avatar
hardy committed
1018 1019 1020 1021
			#count "X2AP-PDU"
			result = re.search('X2AP-PDU', str(line))
			if result is not None:
				x2ap_pdu += 1
hardy's avatar
hardy committed
1022 1023
			#gnb markers logging
			for k in gnb_markers:
1024 1025
				result = re.search(k, line)
				if result is not None:
hardy's avatar
hardy committed
1026
					gnb_markers[k].append(line_cnt)
Remi Hardy's avatar
Remi Hardy committed
1027

1028 1029 1030 1031 1032 1033
			# check whether e/gNB log finishes with "Bye." message
			# Note that it is "=" not "|=" so not only is the regex
			# asking for EOF (\Z) but we also only retain the last
			# line's result
			showedByeMsg = re.search(r'^Bye.\n\Z', str(line), re.MULTILINE) is not None

Gabriele Perrone's avatar
Gabriele Perrone committed
1034
		enb_log_file.close()
1035 1036 1037 1038 1039


		#the following part takes the *_stats.log files as source (not the stdout log file)

		#the datalog config file has to be loaded
1040
		datalog_rt_stats_file=self.datalog_rt_stats_file
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
		if (os.path.isfile(datalog_rt_stats_file)):
			yaml_file=datalog_rt_stats_file
		elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)):
			yaml_file='ci-scripts/'+datalog_rt_stats_file
		else:
			logging.error("Datalog RT stats yaml file cannot be found")
			sys.exit("Datalog RT stats yaml file cannot be found")

		with open(yaml_file,'r') as f:
			datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader)
		rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field  

1053 1054 1055
		if os.path.isfile('./nrL1_stats.log') and os.path.isfile('./nrMAC_stats.log'):
			# don't use CI-nrL1_stats.log, as this will increase the processing time for
			# no reason, we just need the last occurence
hardy's avatar
hardy committed
1056 1057 1058
			nrL1_stats = open('./nrL1_stats.log', 'r')
			nrMAC_stats = open('./nrMAC_stats.log', 'r')
			for line in nrL1_stats.readlines():
1059 1060 1061 1062 1063 1064 1065
				for k in rt_keys:
					result = re.search(k, line)     
					if result is not None:
						#remove 1- all useless char before relevant info  2- trailing char
						tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex
						if tmp!=None: 
							real_time_stats[k]=tmp.group(1)
hardy's avatar
hardy committed
1066
			for line in nrMAC_stats.readlines():
1067 1068 1069 1070 1071 1072 1073 1074 1075
				for k in rt_keys:
					result = re.search(k, line)     
					if result is not None:
						#remove 1- all useless char before relevant info  2- trailing char
						tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex
						if tmp!=None: 
							real_time_stats[k]=tmp.group(1)
			nrL1_stats.close()
			nrMAC_stats.close()
1076 1077
		else:
			logging.debug("NR Stats files for RT analysis not found")
1078 1079 1080 1081 1082

		#stdout log file and stat log files analysis completed
		logging.debug('   File analysis (stdout, stats) completed')

		#post processing depending on the node type
1083 1084 1085 1086 1087
		if not nodeB_prefix_found:
			if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
				nodeB_prefix = 'e'
			else:
				nodeB_prefix = 'g'
hardy's avatar
hardy committed
1088

1089
		if nodeB_prefix == 'g':
1090 1091 1092 1093
			if ulschReceiveOK > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
				htmleNBFailureMsg += statMsg + '\n'
1094 1095 1096 1097
			if gnbRxTxWakeUpFailure > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(gnbRxTxWakeUpFailure) + ' "could not wakeup gNB rxtx process" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
				htmleNBFailureMsg += statMsg + '\n'
1098 1099 1100
			if gnbTxWriteThreadEnabled:
				statMsg = nodeB_prefix + 'NB ran with TX Write thread enabled'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1101 1102 1103 1104 1105 1106 1107 1108
				htmleNBFailureMsg += statMsg + '\n'
			if nrRrcRcfgComplete > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(nrRrcRcfgComplete) + ' "Received NR_RRCReconfigurationComplete from UE" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
				htmleNBFailureMsg += statMsg + '\n'
			if harqFeedbackPast > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(harqFeedbackPast) + ' "HARQ feedback is in the past" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1109
				htmleNBFailureMsg += statMsg + '\n'
hardy's avatar
hardy committed
1110
			#FR1 NSA test : add new markers to make sure gNB is used
1111 1112 1113 1114
			if NSA_RAPROC_PUSCH_check:
				statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : PASS '
				htmlMsg = statMsg+'\n'
			else:
1115
				statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : FAIL or not relevant'
1116 1117 1118
				htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
Remi Hardy's avatar
Remi Hardy committed
1119 1120 1121 1122 1123
			#problem receiving samples log
			statMsg = '[PHY] problem receiving samples msg count =  '+str(pb_receiving_samples_cnt)
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
hardy's avatar
hardy committed
1124
			#gnb markers
1125 1126 1127 1128
			statMsg = 'logfile line count = ' + str(line_cnt)			
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
hardy's avatar
hardy committed
1129 1130
			if len(gnb_markers['SgNBReleaseRequestAcknowledge'])!=0:
				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequestAcknowledge'][0])
1131
			else:
hardy's avatar
hardy committed
1132 1133 1134 1135 1136
				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences' 
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
			statMsg = 'FAILURE = ' + str(len(gnb_markers['FAILURE'])) + ' occurences'
1137 1138 1139
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
hardy's avatar
hardy committed
1140
			statMsg = 'Detected UL Failure on PUSCH = ' + str(len(gnb_markers['Detected UL Failure on PUSCH'])) + ' occurences'
1141 1142 1143
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
Remi Hardy's avatar
Remi Hardy committed
1144

1145 1146
			#ulsch and dlsch statistics and checkers
			#print statistics into html
1147
			if len(dlsch_ulsch_stats)!=0: #check if dictionary is not empty
1148
				#for each dictionary key, generate the msg for html as information
1149
				statMsg=''
1150
				for key in dlsch_ulsch_stats: 
Remi Hardy's avatar
Remi Hardy committed
1151
					statMsg += dlsch_ulsch_stats[key] + '\n' 
1152 1153
					logging.debug(dlsch_ulsch_stats[key])
				htmleNBFailureMsg += statMsg
hardy's avatar
hardy committed
1154

1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
			#checker
			if (len(dlsch_ulsch_stats)!=0) and (len(checkers)!=0):
				if 'd_retx_th' in checkers:
					dlsch_checker_status = list(0 for i in checkers['d_retx_th'])#status 0 / -1
					d_perc_retx = list(0 for i in checkers['d_retx_th'])#results in %

				if 'u_retx_th' in checkers:
					ulsch_checker_status = list(0 for i in checkers['u_retx_th'])
					u_perc_retx = list(0 for i in checkers['u_retx_th'])

				#ul and dl retransmissions checkers
				#NOTICE:  DL and UL regex are different 

				if ('dlsch_rounds' in dlsch_ulsch_stats) and ('d_retx_th' in checkers):
					tmp=re.match(r'^.*dlsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+dlsch_errors\s+(\d+)',dlsch_ulsch_stats['dlsch_rounds'])
					if tmp is not None :
						#captures the different groups from the regex
						retx_data=[float(x) for x in tmp.groups()]
						for i in range(0,len(d_perc_retx)):
							#case where numerator > denumerator with denum ==0 is disregarded, cannot hapen in principle, will lead to 0%
							d_perc_retx[i] = 0 if (retx_data[i] == 0)  else 100*retx_data[i+1]/retx_data[i]
							#treating % > 100 , % > requirement
							if (d_perc_retx[i] > 100) or (d_perc_retx[i] > checkers['d_retx_th'][i]): dlsch_checker_status[i] = -1
					if -1 in dlsch_checker_status:
						DLRetxIssue = True

				if ('ulsch_rounds' in dlsch_ulsch_stats)  and ('u_retx_th' in checkers):
					tmp=re.match(r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+.*,\s+ulsch_errors\s+(\d+)',dlsch_ulsch_stats['ulsch_rounds'])
					if tmp is not None :
						retx_data=[float(x) for x in tmp.groups()]
						for i in range(0,len(d_perc_retx)):
							u_perc_retx[i] = 0 if (retx_data[i] == 0) else 100*retx_data[i+1]/retx_data[i]
							if (u_perc_retx[i] > 100) or (u_perc_retx[i] > checkers['u_retx_th'][i]): ulsch_checker_status[i] = -1
					if -1 in ulsch_checker_status:
						ULRetxIssue = True



hardy's avatar
hardy committed
1193 1194
			#real time statistics
			datalog_rt_stats['Data']={}
1195
			if len(real_time_stats)!=0: #check if dictionary is not empty
hardy's avatar
hardy committed
1196
				for k in real_time_stats:
1197
					tmp=re.match(r'^(?P<metric>.*):\s+(?P<avg>\d+\.\d+) us;\s+(?P<count>\d+);\s+(?P<max>\d+\.\d+) us;',real_time_stats[k])
hardy's avatar
hardy committed
1198 1199 1200 1201
					if tmp is not None:
						metric=tmp.group('metric')
						avg=float(tmp.group('avg'))
						max=float(tmp.group('max'))
Robert Schmidt's avatar
Robert Schmidt committed
1202
						count=int(tmp.group('count'))
1203
						datalog_rt_stats['Data'][metric]=["{:.0f}".format(avg),"{:.0f}".format(max),"{:d}".format(count),"{:.2f}".format(avg/datalog_rt_stats['Ref'][metric])]
1204 1205 1206 1207
				#once all metrics are collected, store the data as a class attribute to build a dedicated HTML table afterward
				self.datalog_rt_stats=datalog_rt_stats
				#check if there is a fail => will render the test as failed
				for k in datalog_rt_stats['Data']:
1208 1209
					if float(datalog_rt_stats['Data'][k][3])> datalog_rt_stats['Threshold'][k]: #condition for fail : avg/ref is greater than the fixed threshold
						logging.debug('\u001B[1;30;43m datalog_rt_stats metric ' + k + '=' + datalog_rt_stats['Data'][k][3] + ' > threshold ' + str(datalog_rt_stats['Threshold'][k]) + ' \u001B[0m')
1210
						RealTimeProcessingIssue = True
1211 1212 1213 1214
			else:
				statMsg = 'No real time stats found in the log file\n'
				logging.debug('No real time stats found in the log file')
				htmleNBFailureMsg += statMsg
1215

1216 1217 1218 1219 1220 1221 1222
			if not showedByeMsg:
				logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB did not show "Bye." message at end, it likely did not stop properly! \u001B[0m')
				htmleNBFailureMsg += 'No Bye. message found, did not stop properly\n'
				global_status = CONST.ENB_SHUTDOWN_NO_BYE
			else:
				logging.debug('"Bye." message found at end.')

1223
		else:
1224 1225 1226 1227 1228
			#Removing UE log
			statMsg = '[MAC] Removing UE msg count =  '+str(removing_ue)
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
hardy's avatar
hardy committed
1229 1230 1231 1232 1233
			#X2AP-PDU log
			statMsg = 'X2AP-PDU msg count =  '+str(x2ap_pdu)
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
1234 1235 1236 1237 1238
			#nsa markers
			statMsg = 'logfile line count = ' + str(line_cnt)			
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
hardy's avatar
hardy committed
1239 1240
			if len(gnb_markers['SgNBReleaseRequest'])!=0:
				statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequest'][0])
1241
			else:
hardy's avatar
hardy committed
1242
				statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences'
1243 1244 1245
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg
hardy's avatar
hardy committed
1246
			statMsg = 'scgFailureInformationNR-r15 = ' + str(len(gnb_markers['scgFailureInformationNR-r15'])) + ' occurences'
1247 1248 1249 1250
			htmlMsg = statMsg+'\n'
			logging.debug(statMsg)
			htmleNBFailureMsg += htmlMsg			

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
		if DLRetxIssue:
			retx_checker_status_str = ''
			for status in dlsch_checker_status : retx_checker_status_str+=str(status)+ ' '
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with too many retransmissions / errors issue in DL ! \u001B[0m')
			logging.debug('\u001B[1;37;41m Status : ' + retx_checker_status_str + ' \u001B[0m')
			htmleNBFailureMsg += 'Fail due to retransmissions / errors issue in DL, status : ' + retx_checker_status_str + '\n'
			global_status = CONST.ENB_RETX_ISSUE

		if ULRetxIssue:
			retx_checker_status_str = ''
			for status in ulsch_checker_status : retx_checker_status_str+=str(status)+ ' '
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with too many retransmissions / errors issue in UL ! \u001B[0m')
			logging.debug('\u001B[1;37;41m Status : ' + retx_checker_status_str + ' \u001B[0m')
			htmleNBFailureMsg += 'Fail due to retransmissions / errors issue in UL, status : ' + retx_checker_status_str + '\n'
			global_status = CONST.ENB_RETX_ISSUE

1267 1268 1269 1270
		if RealTimeProcessingIssue:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with real time processing issue! \u001B[0m')
			htmleNBFailureMsg += 'Fail due to real time processing issue\n'
			global_status = CONST.ENB_REAL_TIME_PROCESSING_ISSUE
Gabriele Perrone's avatar
Gabriele Perrone committed
1271 1272 1273
		if uciStatMsgCount > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1274
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1275 1276 1277
		if pdcpFailure > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1278
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1279 1280 1281
		if ulschFailure > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1282
			htmleNBFailureMsg += statMsg + '\n'
1283 1284 1285 1286 1287 1288 1289 1290
		if ulschAllocateCCEerror > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(ulschAllocateCCEerror) + ' "eNB_dlsch_ulsch_scheduler(); ERROR ALLOCATING CCEs" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			htmleNBFailureMsg += statMsg + '\n'
		if uplinkSegmentsAborted > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(uplinkSegmentsAborted) + ' "uplink segment error 0/2, aborted * segments" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1291 1292 1293
		if dropNotEnoughRBs > 0:
			statMsg = 'eNB showed ' + str(dropNotEnoughRBs) + ' "dropping, not enough RBs" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1294
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1295 1296 1297
		if rrcSetupComplete > 0:
			rrcMsg = nodeB_prefix + 'NB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1298
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1299 1300
			rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1301
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1302 1303 1304
		if rrcReleaseRequest > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1305
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1306 1307 1308
		if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1309
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1310 1311
			rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1312
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1313 1314 1315
		if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1316
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1317 1318
			rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1319
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1320 1321
			rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1322
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1323 1324 1325 1326
		if self.eNBmbmsEnables[int(self.eNB_instance)]:
			if mbmsRequestMsg > 0:
				rrcMsg = 'eNB requested ' + str(mbmsRequestMsg) + ' times the RLC for MBMS USER-PLANE'
				logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1327
				htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1328 1329 1330
		if X2HO_inNbProcedures > 0:
			rrcMsg = 'eNB completed ' + str(X2HO_inNbProcedures) + ' X2 Handover Connection procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1331
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1332 1333 1334
		if X2HO_outNbProcedures > 0:
			rrcMsg = 'eNB completed ' + str(X2HO_outNbProcedures) + ' X2 Handover Release procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1335
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1336 1337 1338 1339 1340 1341
		if self.eNBOptions[int(self.eNB_instance)] != '':
			res1 = re.search('drx_Config_present prSetup', self.eNBOptions[int(self.eNB_instance)])
			if res1 is not None:
				if cdrxActivationMessageCount > 0:
					rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
					logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1342
					htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1343 1344 1345
				else:
					rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration'
					logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
1346
					htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1347 1348 1349
		if rachCanceledProcedure > 0:
			rachMsg = nodeB_prefix + 'NB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
1350
			htmleNBFailureMsg += rachMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1351 1352 1353 1354 1355
		if isRRU:
			if isSlave:
				if slaveReceivesFrameResyncCmd:
					rruMsg = 'Slave RRU received the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;30;43m ' + rruMsg + ' \u001B[0m')
1356
					htmleNBFailureMsg += rruMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1357 1358 1359
				else:
					rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')
1360
					htmleNBFailureMsg += rruMsg + '\n'
1361
					self.prematureExit = True
1362
					global_status = CONST.ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
Gabriele Perrone's avatar
Gabriele Perrone committed
1363 1364
		if foundSegFault:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with a Segmentation Fault! \u001B[0m')
1365
			global_status = CONST.ENB_PROCESS_SEG_FAULT
Gabriele Perrone's avatar
Gabriele Perrone committed
1366 1367
		if foundAssertion:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with an assertion! \u001B[0m')
1368
			htmleNBFailureMsg += msgAssertion
1369
			global_status = CONST.ENB_PROCESS_ASSERTION
Gabriele Perrone's avatar
Gabriele Perrone committed
1370 1371
		if foundRealTimeIssue:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB faced real time issues! \u001B[0m')
1372
			htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues! COUNT = '+ str(foundRealTimeIssue_cnt) +' lines\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1373 1374 1375
		if rlcDiscardBuffer > 0:
			rlcMsg = nodeB_prefix + 'NB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
			logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m')
1376
			htmleNBFailureMsg += rlcMsg + '\n'
1377
			global_status = CONST.ENB_PROCESS_REALTIME_ISSUE
Raphael Defosseux's avatar
Raphael Defosseux committed
1378
		HTML.htmleNBFailureMsg=htmleNBFailureMsg
1379
		# Runtime statistics for console output and HTML
1380 1381
		if runTime != '':
			logging.debug(runTime)
1382 1383 1384 1385
			logging.debug(userTime)
			logging.debug(systemTime)
			logging.debug(maxPhyMemUsage)
			logging.debug(nbContextSwitches)
1386
			self.runtime_stats='<pre>'+runTime + '\n'+ userTime + '\n' + systemTime + '\n' + maxPhyMemUsage + '\n' + nbContextSwitches+'</pre>'
1387
		return global_status