ran.py 46.5 KB
Newer Older
Raphael Defosseux's avatar
Raphael Defosseux committed
1
#/*
Gabriele Perrone's avatar
Gabriele Perrone committed
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

31 32 33 34 35 36 37 38 39 40 41 42 43
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys              # arg
import re               # reg
import logging
import os
import time
from multiprocessing import Process, Lock, SimpleQueue

#-----------------------------------------------------------
# OAI Testing modules
#-----------------------------------------------------------
Gabriele Perrone's avatar
Gabriele Perrone committed
44 45
import sshconnection as SSH
import epc 
46 47
import helpreadme as HELP
import constants as CONST
48
import html
49 50 51 52 53

#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class RANManagement():
Gabriele Perrone's avatar
Gabriele Perrone committed
54

55
	def __init__(self):
Gabriele Perrone's avatar
Gabriele Perrone committed
56 57
		
		self.prematureExit = False
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85
		self.ranRepository = ''
		self.ranBranch = ''
		self.ranAllowMerge = False
		self.ranCommitID = ''
		self.ranTargetBranch = ''
		self.eNBIPAddress = ''
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.eNB1IPAddress = ''
		self.eNB1UserName = ''
		self.eNB1Password = ''
		self.eNB1SourceCodePath = ''
		self.eNB2IPAddress = ''
		self.eNB2UserName = ''
		self.eNB2Password = ''
		self.eNB2SourceCodePath = ''
		self.Build_eNB_args = ''
		self.backgroundBuild = False
		self.backgroundBuildTestId = ['', '', '']
		self.Build_eNB_forced_workspace_cleanup = False
		self.Initialize_eNB_args = ''
		self.air_interface = 'lte'
		self.eNB_instance = ''
		self.eNB_serverId = ''
		self.eNBLogFiles = ['', '', '']
		self.eNBOptions = ['', '', '']
		self.eNBmbmsEnables = [False, False, False]
86 87 88
		self.eNBstatuses = [-1, -1, -1]
		self.flexranCtrlInstalled = False
		self.flexranCtrlStarted = False
89
		self.testCase_id = ''
90 91 92
		self.epcPcapFile = ''
		self.htmlObj = None
		self.epcObj = None
Gabriele Perrone's avatar
Gabriele Perrone committed
93 94

#-----------------------------------------------------------
95
# Setters and Getters on Public members
Gabriele Perrone's avatar
Gabriele Perrone committed
96 97
#-----------------------------------------------------------

98 99 100 101
	def SetHtmlObj(self, obj):
		self.htmlObj = obj
	def SetEpcObj(self, obj):
		self.epcObj = obj
Gabriele Perrone's avatar
Gabriele Perrone committed
102

Gabriele Perrone's avatar
Gabriele Perrone committed
103 104
	def SetflexranCtrlInstalled(self,fxrctin):
		self.flexranCtrlInstalled = fxrctin
105 106
	def GetflexranCtrlInstalled(self):
		return self.flexranCtrlInstalled
Gabriele Perrone's avatar
Gabriele Perrone committed
107 108
	def SetflexranCtrlStarted(self,fxrctst):
		self.flexranCtrlStarted = fxrctst
109 110
	def GetflexranCtrlStarted(self):
		return self.flexranCtrlStarted
Gabriele Perrone's avatar
Gabriele Perrone committed
111 112
	def SetpStatus(self, pSt):
		self.pStatus = pSt
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
	def SetranRepository(self, repository):
		self.ranRepository = repository
	def GetranRepository(self):
		return self.ranRepository
	def SetranBranch(self, branch):
		self.ranBranch = branch
	def GetranBranch(self):
		return self.ranBranch
	def SetranCommitID(self, commitid):
		self.ranCommitID = commitid
	def GetranCommitID(self):
		return self.ranCommitID
	def SeteNB_serverId(self, enbsrvid):
		self.eNB_serverId = enbsrvid
	def GeteNB_serverId(self):
		return self.eNB_serverId
	def SeteNBIPAddress(self, enbip):
		self.eNBIPAddress = enbip
	def GeteNBIPAddress(self):
		return self.eNBIPAddress
	def SeteNBUserName(self, enbusr):
		self.eNBUserName = enbusr
	def GeteNBUserName(self):
		return self.eNBUserName
	def SeteNBPassword(self, enbpw):
		self.eNBPassword = enbpw
	def GeteNBPassword(self):
		return self.eNBPassword
	def SeteNBSourceCodePath(self, enbcodepath):
		self.eNBSourceCodePath = enbcodepath
	def GeteNBSourceCodePath(self):
		return self.eNBSourceCodePath
	def SetranAllowMerge(self, merge):
		self.ranAllowMerge = merge
	def GetranAllowMerge(self):
		return self.ranAllowMerge
	def SetranTargetBranch(self, tbranch):
		self.ranTargetBranch = tbranch
	def GetranTargetBranch(self):
		return self.ranTargetBranch
	def SetBuild_eNB_args(self, enbbuildarg):
		self.Build_eNB_args = enbbuildarg
	def GetBuild_eNB_args(self):
		return self.Build_eNB_args
	def SetInitialize_eNB_args(self, initenbarg):
		self.Initialize_eNB_args = initenbarg
	def GetInitialize_eNB_args(self):
		return self.Initialize_eNB_args
	def SetbackgroundBuild(self, bkbuild):
		self.backgroundBuild = bkbuild
	def GetbackgroundBuild(self):
		return self.backgroundBuild
	def SetbackgroundBuildTestId(self, bkbuildid):
		self.backgroundBuildTestId = bkbuildid
	def GetbackgroundBuildTestId(self):
		return self.backgroundBuildTestId
	def SetBuild_eNB_forced_workspace_cleanup(self, fcdwspclean):
		self.Build_eNB_forced_workspace_cleanup = fcdwspclean
	def GetBuild_eNB_forced_workspace_cleanup(self):
		return self.Build_eNB_forced_workspace_cleanup
	def Setair_interface(self, airif):
		self.air_interface = airif
	def Getair_interface(self):
		return self.air_interface
	def SeteNB_instance(self, enbinst):
		self.eNB_instance = enbinst
	def GeteNB_instance(self):
		return self.eNB_instance

182 183 184 185
	def SeteNBLogFile(self, enblog, idx):
		self.eNBLogFiles[idx] = enblog
	def GeteNBLogFile(self, idx):
		return self.eNBLogFiles[idx]
186

187 188
	def GeteNBmbmsEnable(self, idx):
		return self.eNBmbmsEnables[idx]
189

Gabriele Perrone's avatar
Gabriele Perrone committed
190
	def SeteNB1IPAddress(self,enb1ip):
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
		self.eNB1IPAddress = enb1ip
	def GeteNB1IPAddress(self):
		return self.eNB1IPAddress
	def SeteNB1UserName(self, enb1usr):
		self.eNB1UserName = enb1usr
	def GeteNB1UserName(self):
		return self.eNB1UserName
	def SeteNB1Password(self, enb1pw):
		self.eNB1Password = enb1pw
	def GeteNB1Password(self):
		return self.eNB1Password
	def SeteNB1SourceCodePath(self, enb1codepath):
		self.eNB1SourceCodePath = enb1codepath
	def GeteNB1SourceCodePath(self):
		return self.eNB1SourceCodePath

Gabriele Perrone's avatar
Gabriele Perrone committed
207
	def SeteNB2IPAddress(self, enb2ip):
208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
		self.eNB2IPAddress = enb2ip
	def GeteNB2IPAddress(self):
		return self.eNB2IPAddress
	def SeteNB2UserName(self, enb2usr):
		self.eNB2UserName = enb2usr
	def GeteNB2UserName(self):
		return self.eNB2UserName
	def SeteNB2Password(self, enb2pw):
		self.eNB2Password = enb2pw
	def GeteNB2Password(self):
		return self.eNB2Password
	def SeteNB2SourceCodePath(self, enb2codepath):
		self.eNB2SourceCodePath = enb2codepath
	def GeteNB2SourceCodePath(self):
		return self.eNB2SourceCodePath

Gabriele Perrone's avatar
Gabriele Perrone committed
224 225 226 227
	def SetprematureExit(self, premex):
		self.prematureExit = premex
	def GetprematureExit(self):
		return self.prematureExit
228

Gabriele Perrone's avatar
Gabriele Perrone committed
229 230 231 232
#-----------------------------------------------------------
# RAN management functions
#-----------------------------------------------------------

233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
	def BuildeNB(self):
		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
			GenericHelp(Version)
			sys.exit('Insufficient Parameter')
		if self.eNB_serverId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			GenericHelp(Version)
			sys.exit('Insufficient Parameter')
		mySSH = SSH.SSHConnection()
Gabriele Perrone's avatar
Gabriele Perrone committed
256
		mySSH.open(lIpAddr, lUserName, lPassWord)
257 258 259 260 261 262 263 264 265
		# Check if we build an 5G-NR gNB or an LTE eNB
		result = re.search('--gNB', self.Build_eNB_args)
		if result is not None:
			self.air_interface = 'nr'
		else:
			self.air_interface = 'lte'
		# Worakround for some servers, we need to erase completely the workspace
		if self.Build_eNB_forced_workspace_cleanup:
			mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
266 267 268 269
		if self.htmlObj is not None:
			self.testCase_id = self.htmlObj.GettestCase_id()
		else:
			self.testCase_id = '000000'
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
		# on RedHat/CentOS .git extension is mandatory
		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
		if result is not None:
			full_ran_repo_name = self.ranRepository
		else:
			full_ran_repo_name = self.ranRepository + '.git'
		mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
		# Raphael: here add a check if git clone or git fetch went smoothly
		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
		# Checking the BUILD INFO file
		if not self.backgroundBuild:
			mySSH.command('ls *.txt', '\$', 5)
			result = re.search('LAST_BUILD_INFO', mySSH.getBefore())
			if result is not None:
				mismatch = False
				mySSH.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
				result = re.search(self.ranCommitID, mySSH.getBefore())
				if result is None:
					mismatch = True
				mySSH.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
				if (self.ranAllowMerge):
					result = re.search('YES', mySSH.getBefore())
					if result is None:
						mismatch = True
					mySSH.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
					if self.ranTargetBranch == '':
						result = re.search('develop', mySSH.getBefore())
					else:
						result = re.search(self.ranTargetBranch, mySSH.getBefore())
					if result is None:
						mismatch = True
				else:
					result = re.search('NO', mySSH.getBefore())
					if result is None:
						mismatch = True
				if not mismatch:
					mySSH.close()
310 311
					if self.htmlObj is not None:
						self.htmlObj.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
					return

		mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
		# if the commit ID is provided use it to point to it
		if self.ranCommitID != '':
			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5)
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
		if (self.ranAllowMerge):
			if self.ranTargetBranch == '':
				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
		mySSH.command('source oaienv', '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
		mySSH.command('mkdir -p log', '\$', 5)
		mySSH.command('chmod 777 log', '\$', 5)
		# no need to remove in log (git clean did the trick)
		if self.backgroundBuild:
			mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5)
			mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5)
			mySSH.command('echo ' + lPassWord + ' | sudo -S -E daemon --inherit --unsafe --name=build_enb_daemon --chdir=' + lSourcePath + '/cmake_targets -o ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ./my-lte-softmodem-build.sh', '\$', 5)
			mySSH.close()
337 338
			if self.htmlObj is not None:
				self.htmlObj.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
339
			self.backgroundBuildTestId[int(self.eNB_instance)] = self.testCase_id
340 341
			return
		mySSH.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests|build have failed', 1500)
342
		mySSH.close()
343
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.testCase_id)
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363

	def WaitBuildeNBisFinished(self):
		if self.eNB_serverId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			GenericHelp(Version)
			sys.exit('Insufficient Parameter')
Gabriele Perrone's avatar
Gabriele Perrone committed
364
		mySSH = SSH.SSHConnection()
365 366 367 368 369 370 371 372 373 374 375
		mySSH.open(lIpAddr, lUserName, lPassWord)
		count = 40
		buildOAIprocess = True
		while (count > 0) and buildOAIprocess:
			mySSH.command('ps aux | grep --color=never build_ | grep -v grep', '\$', 3)
			result = re.search('build_oai', mySSH.getBefore())
			if result is None:
				buildOAIprocess = False
			else:
				count -= 1
				time.sleep(30)
376
		mySSH.close()
377 378 379
		self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.backgroundBuildTestId[int(self.eNB_instance)])

	def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId):
380 381
		if self.htmlObj is not None:
			self.htmlObj.SettestCase_id(testcaseId)
382
		mySSH = SSH.SSHConnection()
383
		mySSH.open(lIpAddr, lUserName, lPassWord)
Gabriele Perrone's avatar
Gabriele Perrone committed
384
		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 3)
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430
		mySSH.command('ls ran_build/build', '\$', 3)
		mySSH.command('ls ran_build/build', '\$', 3)
		if self.air_interface == 'nr':
			nodeB_prefix = 'g'
		else:
			nodeB_prefix = 'e'
		buildStatus = True
		result = re.search(self.air_interface + '-softmodem', mySSH.getBefore())
		if result is None:
			buildStatus = False
		else:
			# Generating a BUILD INFO file
			mySSH.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2)
			mySSH.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			if (self.ranAllowMerge):
				mySSH.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				if self.ranTargetBranch == '':
					mySSH.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				else:
					mySSH.command('echo "TGT_BRANCH: ' + self.ranTargetBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			else:
				mySSH.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
		mySSH.command('mkdir -p build_log_' + testcaseId, '\$', 5)
		mySSH.command('mv log/* ' + 'build_log_' + testcaseId, '\$', 5)
		mySSH.command('mv compile_oai_enb.log ' + 'build_log_' + testcaseId, '\$', 5)
		if self.eNB_serverId != '0':
			mySSH.command('cd cmake_targets', '\$', 5)
			mySSH.command('if [ -e tmp_build' + testcaseId + '.zip ]; then rm -f tmp_build' + testcaseId + '.zip; fi', '\$', 5)
			mySSH.command('zip -r -qq tmp_build' + testcaseId + '.zip build_log_' + testcaseId, '\$', 5)
			mySSH.close()
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				os.remove('./tmp_build' + testcaseId + '.zip')
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/tmp_build' + testcaseId + '.zip', '.')
			if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './tmp_build' + testcaseId + '.zip', self.eNBSourceCodePath + '/cmake_targets/.')
				os.remove('./tmp_build' + testcaseId + '.zip')
				mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
				mySSH.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5)
				mySSH.command('unzip -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5)
				mySSH.command('rm -f tmp_build' + testcaseId + '.zip', '\$', 5)
				mySSH.close()
		else:
			mySSH.close()

		if buildStatus:
			logging.info('\u001B[1m Building OAI ' + nodeB_prefix + 'NB Pass\u001B[0m')
431 432
			if self.htmlObj is not None:
				self.htmlObj.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
433 434
		else:
			logging.error('\u001B[1m Building OAI ' + nodeB_prefix + 'NB Failed\u001B[0m')
435 436 437
			if self.htmlObj is not None:
				self.htmlObj.CreateHtmlTestRow(self.Build_eNB_args, 'KO', CONST.ALL_PROCESSES_OK)
				self.htmlObj.CreateHtmlTabFooter(False)
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458
			sys.exit(1)

	def InitializeeNB(self):
		if self.eNB_serverId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			GenericHelp(Version)
			sys.exit('Insufficient Parameter')
459 460 461 462 463

		if self.htmlObj is not None:
			self.testCase_id = self.htmlObj.GettestCase_id()
		else:
			self.testCase_id = '000000'
Gabriele Perrone's avatar
Gabriele Perrone committed
464
		mySSH = SSH.SSHConnection()
Gabriele Perrone's avatar
Gabriele Perrone committed
465
		
Gabriele Perrone's avatar
Gabriele Perrone committed
466
		if (self.pStatus < 0):
467 468 469
			if self.htmlObj is not None:
				self.htmlObj.CreateHtmlTestRow(self.Initialize_eNB_args, 'KO', self.pStatus)
				self.htmlObj.CreateHtmlTabFooter(False)
470 471 472
			sys.exit(1)
		# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
473 474 475 476 477
		if (result is not None) and (self.epcObj is not None):
			localEpcIpAddr = self.epcObj.GetIPAddress()
			localEpcUserName = self.epcObj.GetUserName()
			localEpcPassword = self.epcObj.GetPassword()
			mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
478 479 480 481 482
			mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
			result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore())
			if result is not None:
				eth_interface = result.group('eth_interface')
				logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
483 484 485
				self.epcPcapFile = 'enb_' + self.testCase_id + '_s1log.pcap'
				mySSH.command('echo ' + localEpcPassword + ' | sudo -S rm -f /tmp/' + self.epcPcapFile , '\$', 5)
				mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5)
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523
			mySSH.close()
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		# Initialize_eNB_args usually start with -O and followed by the location in repository
		full_config_file = self.Initialize_eNB_args.replace('-O ','')
		extra_options = ''
		extIdx = full_config_file.find('.conf')
		if (extIdx > 0):
			extra_options = full_config_file[extIdx + 5:]
			# if tracer options is on, compiling and running T Tracer
			result = re.search('T_stdout', str(extra_options))
			if result is not None:
				logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
				mySSH.command('cd common/utils/T/tracer', '\$', 5)
				mySSH.command('make', '\$', 10)
				mySSH.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', lUserName, 5)
				mySSH.command('cd ' + lSourcePath, '\$', 5)
			full_config_file = full_config_file[:extIdx + 5]
			config_path, config_file = os.path.split(full_config_file)
		else:
			sys.exit('Insufficient Parameter')
		ci_full_config_file = config_path + '/ci-' + config_file
		rruCheck = False
		result = re.search('^rru|^rcc|^du.band', str(config_file))
		if result is not None:
			rruCheck = True
		# do not reset board twice in IF4.5 case
		result = re.search('^rru|^enb|^du.band', str(config_file))
		if result is not None:
			mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60)
			result = re.search('type: b200', mySSH.getBefore())
			if result is not None:
				logging.debug('Found a B2xx device --> resetting it')
				mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
				# Reloading FGPA bin firmware
				mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60)
		# Make a copy and adapt to EPC / eNB IP addresses
		mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
524 525 526
		if self.epcObj is not None:
			localEpcIpAddr = self.epcObj.GetIPAddress()
			mySSH.command('sed -i -e \'s/CI_MME_IP_ADDR/' + localEpcIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
		mySSH.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RRU1_IP_ADDR/' + self.eNB1IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		mySSH.command('sed -i -e \'s/CI_RRU2_IP_ADDR/' + self.eNB2IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		if self.flexranCtrlInstalled and self.flexranCtrlStarted:
			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "yes";/\' ' + ci_full_config_file, '\$', 2);
		else:
			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "no";/\' ' + ci_full_config_file, '\$', 2);
		self.eNBmbmsEnables[int(self.eNB_instance)] = False
		mySSH.command('grep enable_enb_m2 ' + ci_full_config_file, '\$', 2);
		result = re.search('yes', mySSH.getBefore())
		if result is not None:
			self.eNBmbmsEnables[int(self.eNB_instance)] = True
			logging.debug('\u001B[1m MBMS is enabled on this eNB\u001B[0m')
		result = re.search('noS1', str(self.Initialize_eNB_args))
		eNBinNoS1 = False
		if result is not None:
			eNBinNoS1 = True
			logging.debug('\u001B[1m eNB is in noS1 configuration \u001B[0m')
		# Launch eNB with the modified config file
		mySSH.command('source oaienv', '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
		mySSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface + '-softmodem -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)
		mySSH.command('hostnamectl','\$', 5)
		result = re.search('CentOS Linux 7', mySSH.getBefore())
		if result is not None:
			mySSH.command('echo $USER; nohup sudo ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10)
		else:
			mySSH.command('echo ' + lPassWord + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + lSourcePath + '/cmake_targets -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log'
		if extra_options != '':
			self.eNBOptions[int(self.eNB_instance)] = extra_options
		time.sleep(6)
		doLoop = True
		loopCounter = 20
		enbDidSync = False
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
				# In case of T tracer recording, we may need to kill it
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
				if result is not None:
					mySSH.command('killall --signal SIGKILL record', '\$', 5)
				mySSH.close()
				doLoop = False
				logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
575 576
				if self.htmlObj is not None:
					self.htmlObj.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', CONST.ALL_PROCESSES_OK)
577 578
				# In case of T tracer recording, we need to kill tshark on EPC side
				result = re.search('T_stdout', str(self.Initialize_eNB_args))
579 580 581 582 583
				if (result is not None) and (self.epcObj is not None):
					localEpcIpAddr = self.epcObj.GetIPAddress()
					localEpcUserName = self.epcObj.GetUserName()
					localEpcPassword = self.epcObj.GetPassword()
					mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
584
					logging.debug('\u001B[1m Stopping tshark \u001B[0m')
585 586
					mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
					if self.epcPcapFile  != '':
587
						time.sleep(0.5)
588
						mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
589 590
					mySSH.close()
					time.sleep(1)
591 592
					if self.epcPcapFile != '':
						copyin_res = mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
593
						if (copyin_res == 0):
594
							mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628
				self.prematureExit = True
				return
			else:
				mySSH.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting"', '\$', 4)
				if rruCheck:
					result = re.search('wait RUs', mySSH.getBefore())
				else:
					result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore())
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
					enbDidSync = True
					time.sleep(10)

		if enbDidSync and eNBinNoS1:
			mySSH.command('ifconfig oaitun_enb1', '\$', 4)
			mySSH.command('ifconfig oaitun_enb1', '\$', 4)
			result = re.search('inet addr:1|inet 1', mySSH.getBefore())
			if result is not None:
				logging.debug('\u001B[1m oaitun_enb1 interface is mounted and configured\u001B[0m')
			else:
				logging.error('\u001B[1m oaitun_enb1 interface is either NOT mounted or NOT configured\u001B[0m')
			if self.eNBmbmsEnables[int(self.eNB_instance)]:
				mySSH.command('ifconfig oaitun_enm1', '\$', 4)
				result = re.search('inet addr', mySSH.getBefore())
				if result is not None:
					logging.debug('\u001B[1m oaitun_enm1 interface is mounted and configured\u001B[0m')
				else:
					logging.error('\u001B[1m oaitun_enm1 interface is either NOT mounted or NOT configured\u001B[0m')
		if enbDidSync:
			self.eNBstatuses[int(self.eNB_instance)] = int(self.eNB_serverId)

		mySSH.close()
629 630
		if self.htmlObj is not None:
			self.htmlObj.CreateHtmlTestRow('-O ' + config_file + extra_options, 'OK', CONST.ALL_PROCESSES_OK)
631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
		logging.debug('\u001B[1m Initialize eNB Completed\u001B[0m')

	def CheckeNBProcess(self, status_queue):
		try:
			# At least the instance 0 SHALL be on!
			if self.eNBstatuses[0] == 0:
				lIpAddr = self.eNBIPAddress
				lUserName = self.eNBUserName
				lPassWord = self.eNBPassword
			elif self.eNBstatuses[0] == 1:
				lIpAddr = self.eNB1IPAddress
				lUserName = self.eNB1UserName
				lPassWord = self.eNB1Password
			elif self.eNBstatuses[0] == 2:
				lIpAddr = self.eNB2IPAddress
				lUserName = self.eNB2UserName
				lPassWord = self.eNB2Password
			else:
				lIpAddr = self.eNBIPAddress
				lUserName = self.eNBUserName
				lPassWord = self.eNBPassword
Gabriele Perrone's avatar
Gabriele Perrone committed
652
			mySSH = SSH.SSHConnection()
653 654 655 656 657
			mySSH.open(lIpAddr, lUserName, lPassWord)
			mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + self.air_interface + '-softmodem | grep -v grep', '\$', 5)
			result = re.search(self.air_interface + '-softmodem', mySSH.getBefore())
			if result is None:
				logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
Gabriele Perrone's avatar
Gabriele Perrone committed
658
				status_queue.put(CONST.ENB_PROCESS_FAILED)
659
			else:
Gabriele Perrone's avatar
Gabriele Perrone committed
660
				status_queue.put(CONST.ENB_PROCESS_OK)
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
			mySSH.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def TerminateeNB(self):
		if self.eNB_serverId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			GenericHelp(Version)
			sys.exit('Insufficient Parameter')
Gabriele Perrone's avatar
Gabriele Perrone committed
684
		mySSH = SSH.SSHConnection()
685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
		if self.air_interface == 'lte':
			nodeB_prefix = 'e'
		else:
			nodeB_prefix = 'g'
		mySSH.command('stdbuf -o0  ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
		result = re.search('-softmodem', mySSH.getBefore())
		if result is not None:
			mySSH.command('echo ' + lPassWord + ' | sudo -S daemon --name=enb' + str(self.eNB_instance) + '_daemon --stop', '\$', 5)
			mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT -r .*-softmodem || true', '\$', 5)
			time.sleep(10)
			mySSH.command('stdbuf -o0  ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
			result = re.search('-softmodem', mySSH.getBefore())
			if result is not None:
				mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem || true', '\$', 5)
				time.sleep(5)
		mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
		mySSH.close()
		# If tracer options is on, stopping tshark on EPC side
		result = re.search('T_stdout', str(self.Initialize_eNB_args))
706 707 708 709 710
		if (result is not None) and (self.epcObj is not None):
			localEpcIpAddr = self.epcObj.GetIPAddress()
			localEpcUserName = self.epcObj.GetUserName()
			localEpcPassword = self.epcObj.GetPassword()
			mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
711
			logging.debug('\u001B[1m Stopping tshark \u001B[0m')
712
			mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
713
			time.sleep(1)
714 715 716 717
			if self.epcPcapFile != '':
				mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
				mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
				mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
			mySSH.close()
			logging.debug('\u001B[1m Replaying RAW record file\u001B[0m')
			mySSH.open(lIpAddr, lUserName, lPassWord)
			mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5)
			enbLogFile = self.eNBLogFiles[int(self.eNB_instance)]
			raw_record_file = enbLogFile.replace('.log', '_record.raw')
			replay_log_file = enbLogFile.replace('.log', '_replay.log')
			extracted_txt_file = enbLogFile.replace('.log', '_extracted_messages.txt')
			extracted_log_file = enbLogFile.replace('.log', '_extracted_messages.log')
			mySSH.command('./extract_config -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + extracted_txt_file, '\$', 5)
			mySSH.command('echo $USER; nohup ./replay -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', lUserName, 5)
			mySSH.command('./textlog -d ' +  lSourcePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + lSourcePath + '/cmake_targets/' + extracted_log_file, '\$', 5)
			mySSH.close()
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + extracted_log_file, '.')
			logging.debug('\u001B[1m Analyzing eNB replay logfile \u001B[0m')
			logStatus = self.AnalyzeLogFile_eNB(extracted_log_file)
734 735
			if self.htmlObj is not None:
				self.htmlObj.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
736 737 738 739 740 741 742 743 744 745 746
			self.eNBLogFiles[int(self.eNB_instance)] = ''
		else:
			analyzeFile = False
			if self.eNBLogFiles[int(self.eNB_instance)] != '':
				analyzeFile = True
				fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
				self.eNBLogFiles[int(self.eNB_instance)] = ''
			if analyzeFile:
				copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
				if (copyin_res == -1):
					logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
747
					if self.htmlObj is not None:
748
						self.htmlObj.SetHmleNBFailureMsg('Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!')
749
						self.htmlObj.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
750 751 752 753 754 755 756
					self.eNBmbmsEnables[int(self.eNB_instance)] = False
					return
				if self.eNB_serverId != '0':
					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze)
				logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze)
				if (logStatus < 0):
757 758
					if self.htmlObj is not None:
						self.htmlObj.CreateHtmlTestRow('N/A', 'KO', logStatus)
759 760 761 762
					self.preamtureExit = True
					self.eNBmbmsEnables[int(self.eNB_instance)] = False
					return
				else:
763 764
					if self.htmlObj is not None:
						self.htmlObj.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
765
			else:
766 767
				if self.htmlObj is not None:
					self.htmlObj.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
768 769 770 771
		self.eNBmbmsEnables[int(self.eNB_instance)] = False
		self.eNBstatuses[int(self.eNB_instance)] = -1

	def LogCollecteNB(self):
Gabriele Perrone's avatar
Gabriele Perrone committed
772
		mySSH = SSH.SSHConnection()
773 774 775 776 777 778 779
		mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5)
		mySSH.command('cd cmake_targets', '\$', 5)
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 60)
		mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 5)
		mySSH.close()
Gabriele Perrone's avatar
Gabriele Perrone committed
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802

	def AnalyzeLogFile_eNB(self, eNBlogFile):
		if (not os.path.isfile('./' + eNBlogFile)):
			return -1
		enb_log_file = open('./' + eNBlogFile, 'r')
		exitSignalReceived = False
		foundAssertion = False
		msgAssertion = ''
		msgLine = 0
		foundSegFault = False
		foundRealTimeIssue = False
		rrcSetupComplete = 0
		rrcReleaseRequest = 0
		rrcReconfigRequest = 0
		rrcReconfigComplete = 0
		rrcReestablishRequest = 0
		rrcReestablishComplete = 0
		rrcReestablishReject = 0
		rlcDiscardBuffer = 0
		rachCanceledProcedure = 0
		uciStatMsgCount = 0
		pdcpFailure = 0
		ulschFailure = 0
803
		ulschReceiveOK = 0
Gabriele Perrone's avatar
Gabriele Perrone committed
804 805 806
		cdrxActivationMessageCount = 0
		dropNotEnoughRBs = 0
		mbmsRequestMsg = 0
807
		htmleNBFailureMsg = ''
Gabriele Perrone's avatar
Gabriele Perrone committed
808 809 810
		isRRU = False
		isSlave = False
		slaveReceivesFrameResyncCmd = False
Gabriele Perrone's avatar
Gabriele Perrone committed
811
		X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
812 813 814
		X2HO_inNbProcedures = 0
		X2HO_outNbProcedures = 0
		for line in enb_log_file.readlines():
Gabriele Perrone's avatar
Gabriele Perrone committed
815
			if X2HO_state == CONST.X2_HO_REQ_STATE__IDLE:
Gabriele Perrone's avatar
Gabriele Perrone committed
816 817
				result = re.search('target eNB Receives X2 HO Req X2AP_HANDOVER_REQ', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
818
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ
Gabriele Perrone's avatar
Gabriele Perrone committed
819 820
				result = re.search('source eNB receives the X2 HO ACK X2AP_HANDOVER_REQ_ACK', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
821 822
					X2HO_state = CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ:
Gabriele Perrone's avatar
Gabriele Perrone committed
823 824
				result = re.search('Received LTE_RRCConnectionReconfigurationComplete from UE', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
825 826
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE:
Gabriele Perrone's avatar
Gabriele Perrone committed
827 828
				result = re.search('issue rrc_eNB_send_PATH_SWITCH_REQ', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
829 830
					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ
			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ:
Gabriele Perrone's avatar
Gabriele Perrone committed
831 832
				result = re.search('received path switch ack S1AP_PATH_SWITCH_REQ_ACK', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
833
					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
834
					X2HO_inNbProcedures += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
835
			if X2HO_state == CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK:
Gabriele Perrone's avatar
Gabriele Perrone committed
836 837
				result = re.search('source eNB receives the X2 UE CONTEXT RELEASE X2AP_UE_CONTEXT_RELEASE', str(line))
				if result is not None:
Gabriele Perrone's avatar
Gabriele Perrone committed
838
					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
Gabriele Perrone's avatar
Gabriele Perrone committed
839 840 841 842 843 844 845 846 847
					X2HO_outNbProcedures += 1

			if self.eNBOptions[int(self.eNB_instance)] != '':
				res1 = re.search('max_rxgain (?P<requested_option>[0-9]+)', self.eNBOptions[int(self.eNB_instance)])
				res2 = re.search('max_rxgain (?P<applied_option>[0-9]+)',  str(line))
				if res1 is not None and res2 is not None:
					requested_option = int(res1.group('requested_option'))
					applied_option = int(res2.group('applied_option'))
					if requested_option == applied_option:
848
						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ok-circle"></span> Command line option(s) correctly applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
849
					else:
850
						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ban-circle"></span> Command line option(s) NOT applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915
			result = re.search('Exiting OAI softmodem', str(line))
			if result is not None:
				exitSignalReceived = True
			result = re.search('[Ss]egmentation [Ff]ault', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Cc]ore [dD]ump', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('./ran_build/build/lte-softmodem', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Aa]ssertion', str(line))
			if result is not None and not exitSignalReceived:
				foundAssertion = True
			result = re.search('LLL', str(line))
			if result is not None and not exitSignalReceived:
				foundRealTimeIssue = True
			if foundAssertion and (msgLine < 3):
				msgLine += 1
				msgAssertion += str(line)
			result = re.search('Setting function for RU', str(line))
			if result is not None:
				isRRU = True
			if isRRU:
				result = re.search('RU 0 is_slave=yes', str(line))
				if result is not None:
					isSlave = True
				if isSlave:
					result = re.search('Received RRU_frame_resynch command', str(line))
					if result is not None:
						slaveReceivesFrameResyncCmd = True
			result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line))
			if result is not None:
				rrcSetupComplete += 1
			result = re.search('Generate LTE_RRCConnectionRelease|Generate RRCConnectionRelease', str(line))
			if result is not None:
				rrcReleaseRequest += 1
			result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line))
			if result is not None:
				rrcReconfigRequest += 1
			result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line))
			if result is not None:
				rrcReconfigComplete += 1
			result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line))
			if result is not None:
				rrcReestablishRequest += 1
			result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line))
			if result is not None:
				rrcReestablishComplete += 1
			result = re.search('LTE_RRCConnectionReestablishmentReject', str(line))
			if result is not None:
				rrcReestablishReject += 1
			result = re.search('CDRX configuration activated after RRC Connection', str(line))
			if result is not None:
				cdrxActivationMessageCount += 1
			result = re.search('uci->stat', str(line))
			if result is not None:
				uciStatMsgCount += 1
			result = re.search('PDCP.*Out of Resources.*reason', str(line))
			if result is not None:
				pdcpFailure += 1
			result = re.search('ULSCH in error in round', str(line))
			if result is not None:
				ulschFailure += 1
916 917 918
			result = re.search('ULSCH received ok', str(line))
			if result is not None:
				ulschReceiveOK += 1
Gabriele Perrone's avatar
Gabriele Perrone committed
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933
			result = re.search('BAD all_segments_received', str(line))
			if result is not None:
				rlcDiscardBuffer += 1
			result = re.search('Canceled RA procedure for UE rnti', str(line))
			if result is not None:
				rachCanceledProcedure += 1
			result = re.search('dropping, not enough RBs', str(line))
			if result is not None:
				dropNotEnoughRBs += 1
			if self.eNBmbmsEnables[int(self.eNB_instance)]:
				result = re.search('MBMS USER-PLANE.*Requesting.*bytes from RLC', str(line))
				if result is not None:
					mbmsRequestMsg += 1
		enb_log_file.close()
		logging.debug('   File analysis completed')
934
		if self.air_interface == 'lte':
Gabriele Perrone's avatar
Gabriele Perrone committed
935 936 937
			nodeB_prefix = 'e'
		else:
			nodeB_prefix = 'g'
938 939 940 941 942
		if self.air_interface == 'nr':
			if ulschReceiveOK > 0:
				statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)'
				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
				htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
943 944 945
		if uciStatMsgCount > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
946
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
947 948 949
		if pdcpFailure > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
950
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
951 952 953
		if ulschFailure > 0:
			statMsg = nodeB_prefix + 'NB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
954
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
955 956 957
		if dropNotEnoughRBs > 0:
			statMsg = 'eNB showed ' + str(dropNotEnoughRBs) + ' "dropping, not enough RBs" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
958
			htmleNBFailureMsg += statMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
959 960 961
		if rrcSetupComplete > 0:
			rrcMsg = nodeB_prefix + 'NB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
962
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
963 964
			rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
965
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
966 967 968
		if rrcReleaseRequest > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
969
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
970 971 972
		if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
973
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
974 975
			rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
976
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
977 978 979
		if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0:
			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
980
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
981 982
			rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
983
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
984 985
			rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
986
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
987 988 989 990
		if self.eNBmbmsEnables[int(self.eNB_instance)]:
			if mbmsRequestMsg > 0:
				rrcMsg = 'eNB requested ' + str(mbmsRequestMsg) + ' times the RLC for MBMS USER-PLANE'
				logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
991
				htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
992 993 994
		if X2HO_inNbProcedures > 0:
			rrcMsg = 'eNB completed ' + str(X2HO_inNbProcedures) + ' X2 Handover Connection procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
995
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
996 997 998
		if X2HO_outNbProcedures > 0:
			rrcMsg = 'eNB completed ' + str(X2HO_outNbProcedures) + ' X2 Handover Release procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
999
			htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1000 1001 1002 1003 1004 1005
		if self.eNBOptions[int(self.eNB_instance)] != '':
			res1 = re.search('drx_Config_present prSetup', self.eNBOptions[int(self.eNB_instance)])
			if res1 is not None:
				if cdrxActivationMessageCount > 0:
					rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
					logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
1006
					htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1007 1008 1009
				else:
					rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration'
					logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
1010
					htmleNBFailureMsg += rrcMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1011 1012 1013
		if rachCanceledProcedure > 0:
			rachMsg = nodeB_prefix + 'NB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
			logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
1014
			htmleNBFailureMsg += rachMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1015 1016 1017 1018 1019
		if isRRU:
			if isSlave:
				if slaveReceivesFrameResyncCmd:
					rruMsg = 'Slave RRU received the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;30;43m ' + rruMsg + ' \u001B[0m')
1020
					htmleNBFailureMsg += rruMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1021 1022 1023
				else:
					rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
					logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')
1024
					htmleNBFailureMsg += rruMsg + '\n'
Gabriele Perrone's avatar
Gabriele Perrone committed
1025
					self.prematureExit(True)
Gabriele Perrone's avatar
Gabriele Perrone committed
1026
					return CONST.ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
Gabriele Perrone's avatar
Gabriele Perrone committed
1027 1028
		if foundSegFault:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with a Segmentation Fault! \u001B[0m')
1029 1030
			if self.htmlObj is not None:
				self.htmlObj.SetHmleNBFailureMsg(htmleNBFailureMsg)
Gabriele Perrone's avatar
Gabriele Perrone committed
1031
			return CONST.ENB_PROCESS_SEG_FAULT
Gabriele Perrone's avatar
Gabriele Perrone committed
1032 1033
		if foundAssertion:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with an assertion! \u001B[0m')
1034 1035 1036
			htmleNBFailureMsg += msgAssertion
			if self.htmlObj is not None:
				self.htmlObj.SetHmleNBFailureMsg(htmleNBFailureMsg)
Gabriele Perrone's avatar
Gabriele Perrone committed
1037
			return CONST.ENB_PROCESS_ASSERTION
Gabriele Perrone's avatar
Gabriele Perrone committed
1038 1039
		if foundRealTimeIssue:
			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB faced real time issues! \u001B[0m')
1040
			htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues!\n'
Raphael Defosseux's avatar
Raphael Defosseux committed
1041
			#return CONST.ENB_PROCESS_REALTIME_ISSUE
Gabriele Perrone's avatar
Gabriele Perrone committed
1042 1043 1044
		if rlcDiscardBuffer > 0:
			rlcMsg = nodeB_prefix + 'NB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
			logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m')
1045 1046 1047
			htmleNBFailureMsg += rlcMsg + '\n'
			if self.htmlObj is not None:
				self.htmlObj.SetHmleNBFailureMsg(htmleNBFailureMsg)
Gabriele Perrone's avatar
Gabriele Perrone committed
1048
			return CONST.ENB_PROCESS_REALTIME_ISSUE
1049 1050
		if self.htmlObj is not None:
			self.htmlObj.SetHmleNBFailureMsg(htmleNBFailureMsg)
Gabriele Perrone's avatar
Gabriele Perrone committed
1051
		return 0