main.py 94.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

#-----------------------------------------------------------
# Version
#-----------------------------------------------------------
Version = '0.1'

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
#-----------------------------------------------------------
# Constants
#-----------------------------------------------------------
ALL_PROCESSES_OK = 0
ENB_PROCESS_FAILED = -1
ENB_PROCESS_OK = +1
ENB_PROCESS_SEG_FAULT = -11
ENB_PROCESS_ASSERTION = -12
ENB_PROCESS_REALTIME_ISSUE = -13
HSS_PROCESS_FAILED = -2
HSS_PROCESS_OK = +2
MME_PROCESS_FAILED = -3
MME_PROCESS_OK = +3
SPGW_PROCESS_FAILED = -4
SPGW_PROCESS_OK = +4
UE_IP_ADDRESS_ISSUE = -5

53 54 55 56 57 58 59 60 61 62 63 64
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys		# arg
import re		# reg
import pexpect		# pexpect
import time		# sleep
import os
import xml.etree.ElementTree as ET
import logging
import datetime
import signal
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
65
from multiprocessing import Process, Lock, SimpleQueue
66
logging.basicConfig(
67 68
	level=logging.DEBUG,
	format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s"
69 70 71 72 73 74 75 76 77 78
)

#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class SSHConnection():
	def __init__(self):
		self.eNBIPAddress = ''
		self.eNBRepository = ''
		self.eNBBranch = ''
Raphael Defosseux's avatar
Raphael Defosseux committed
79
		self.eNB_AllowMerge = False
80
		self.eNBCommitID = ''
81
		self.eNBTargetBranch = ''
82 83 84 85 86 87 88 89 90 91 92 93
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.EPCIPAddress = ''
		self.EPCUserName = ''
		self.EPCPassword = ''
		self.EPCSourceCodePath = ''
		self.EPCType = ''
		self.ADBIPAddress = ''
		self.ADBUserName = ''
		self.ADBPassword = ''
		self.testCase_id = ''
94
		self.testXMLfile = ''
95 96 97
		self.desc = ''
		self.Build_eNB_args = ''
		self.Initialize_eNB_args = ''
98
		self.eNBLogFile = ''
99
		self.eNB_instance = ''
100 101 102 103
		self.ping_args = ''
		self.ping_packetloss_threshold = ''
		self.iperf_args = ''
		self.iperf_packetloss_threshold = ''
104
		self.iperf_profile = ''
105 106
		self.UEDevices = []
		self.UEIPAddresses = []
Raphael Defosseux's avatar
Raphael Defosseux committed
107 108 109 110
		self.htmlFile = ''
		self.htmlHeaderCreated = False
		self.htmlFooterCreated = False
		self.htmlUEConnected = 0
111
		self.htmleNBFailureMsg = ''
112

113
	def open(self, ipaddress, username, password):
114 115 116 117 118
		count = 0
		connect_status = False
		while count < 4:
			self.ssh = pexpect.spawn('ssh', [username + '@' + ipaddress], timeout = 5)
			self.sshresponse = self.ssh.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', 'Last login', pexpect.EOF, pexpect.TIMEOUT])
119
			if self.sshresponse == 0:
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
				self.ssh.sendline('yes')
				self.ssh.expect('password:')
				self.ssh.sendline(password)
				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if self.sshresponse == 0:
					count = 10
					connect_status = True
				else:
					logging.debug('self.sshresponse = ' + str(self.sshresponse))
			elif self.sshresponse == 1:
				self.ssh.sendline(password)
				self.sshresponse = self.ssh.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
				if self.sshresponse == 0:
					count = 10
					connect_status = True
				else:
					logging.debug('self.sshresponse = ' + str(self.sshresponse))
			elif self.sshresponse == 2:
				# Checking if we are really on the remote client defined by its IP address
				self.command('stdbuf -o0 ifconfig | egrep --color=never "inet addr:"', '\$', 5)
				result = re.search(str(ipaddress), str(self.ssh.before))
				if result is None:
					self.close()
				else:
					count = 10
					connect_status = True
146
			else:
147 148
				# debug output
				logging.debug(str(self.ssh.before))
149
				logging.debug('self.sshresponse = ' + str(self.sshresponse))
Raphael Defosseux's avatar
Raphael Defosseux committed
150 151 152
			# adding a tempo when failure
			if not connect_status:
				time.sleep(1)
153 154
			count += 1
		if connect_status:
155 156
			pass
		else:
157
			sys.exit('SSH Connection Failed')
158

159 160 161 162 163 164
	def command(self, commandline, expectedline, timeout):
		logging.debug(commandline)
		self.ssh.timeout = timeout
		self.ssh.sendline(commandline)
		self.sshresponse = self.ssh.expect([expectedline, pexpect.EOF, pexpect.TIMEOUT])
		if self.sshresponse == 0:
Raphael Defosseux's avatar
Raphael Defosseux committed
165
			return 0
166 167 168 169 170 171 172
		elif self.sshresponse == 1:
			logging.debug('\u001B[1;37;41m Unexpected EOF \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
			sys.exit(self.sshresponse)
		elif self.sshresponse == 2:
			logging.debug('\u001B[1;37;41m Unexpected TIMEOUT \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
Raphael Defosseux's avatar
Raphael Defosseux committed
173 174 175 176 177
			result = re.search('ping |iperf ', str(commandline))
			if result is None:
				sys.exit(self.sshresponse)
			else:
				return -1
178 179 180 181
		else:
			logging.debug('\u001B[1;37;41m Unexpected Others \u001B[0m')
			logging.debug('Expected Line : ' + expectedline)
			sys.exit(self.sshresponse)
182

183 184 185 186 187 188 189 190 191 192
	def close(self):
		self.ssh.timeout = 5
		self.ssh.sendline('exit')
		self.sshresponse = self.ssh.expect([pexpect.EOF, pexpect.TIMEOUT])
		if self.sshresponse == 0:
			pass
		elif self.sshresponse == 1:
			logging.debug('\u001B[1;37;41m Unexpected TIMEOUT \u001B[0m')
		else:
			logging.debug('\u001B[1;37;41m Unexpected Others \u001B[0m')
193

194
	def copyin(self, ipaddress, username, password, source, destination):
Raphael Defosseux's avatar
Raphael Defosseux committed
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
		logging.debug('scp '+ username + '@' + ipaddress + ':' + source + ' ' + destination)
		scp_spawn = pexpect.spawn('scp '+ username + '@' + ipaddress + ':' + source + ' ' + destination, timeout = 5)
		scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
		if scp_response == 0:
			scp_spawn.sendline('yes')
			scp_spawn.expect('password:')
			scp_spawn.sendline(password)
			scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
			if scp_response == 0:
				pass
			else:
				logging.debug('1 - scp_response = ' + str(scp_response))
				sys.exit('SCP failed')
		elif scp_response == 1:
			scp_spawn.sendline(password)
			scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
			if scp_response == 0 or scp_response == 3:
				pass
			else:
				logging.debug('2 - scp_response = ' + str(scp_response))
				sys.exit('SCP failed')
		elif scp_response == 2:
			pass
		else:
			logging.debug('3 - scp_response = ' + str(scp_response))
			sys.exit('SCP failed')

222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
	def copyout(self, ipaddress, username, password, source, destination):
		logging.debug('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination)
		scp_spawn = pexpect.spawn('scp ' + source + ' ' + username + '@' + ipaddress + ':' + destination, timeout = 5)
		scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT])
		if scp_response == 0:
			scp_spawn.sendline('yes')
			scp_spawn.expect('password:')
			scp_spawn.sendline(password)
			scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
			if scp_response == 0:
				pass
			else:
				logging.debug('1 - scp_response = ' + str(scp_response))
				sys.exit('SCP failed')
		elif scp_response == 1:
			scp_spawn.sendline(password)
			scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT])
			if scp_response == 0 or scp_response == 3:
				pass
			else:
				logging.debug('2 - scp_response = ' + str(scp_response))
				sys.exit('SCP failed')
		elif scp_response == 2:
			pass
		else:
			logging.debug('3 - scp_response = ' + str(scp_response))
			sys.exit('SCP failed')

250 251 252 253 254 255 256
	def BuildeNB(self):
		if self.eNBIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.eNBSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('mkdir -p ' + self.eNBSourceCodePath, '\$', 5)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
257 258
		self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.eNBRepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600)
		# Raphael: here add a check if git clone or git fetch went smoothly
259 260
		self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		self.command('git config user.name "OAI Jenkins"', '\$', 5)
261
		self.command('echo ' + self.eNBPassword + ' | sudo -S git clean -x -d -ff', '\$', 30)
262 263 264 265 266
		# if the commit ID is provided use it to point to it
		if self.eNBCommitID != '':
			self.command('git checkout -f ' + self.eNBCommitID, '\$', 5)
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
Raphael Defosseux's avatar
Raphael Defosseux committed
267
		if (self.eNB_AllowMerge):
268 269 270 271 272 273
			if self.eNBTargetBranch == '':
				if (self.eNBBranch != 'develop') and (self.eNBBranch != 'origin/develop'):
					self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
			else:
				logging.debug('Merging with the target branch: ' + self.eNBTargetBranch)
				self.command('git merge --ff origin/' + self.eNBTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
274 275
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
276 277
		self.command('mkdir -p log', '\$', 5)
		self.command('chmod 777 log', '\$', 5)
278
		# no need to remove in log (git clean did the trick)
279
		self.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee -a compile_oai_enb.log', 'Bypassing the Tests', 600)
280
		self.command('mkdir -p build_log_' + SSH.testCase_id, '\$', 5)
281 282 283 284
		self.command('mv log/* ' + 'build_log_' + SSH.testCase_id, '\$', 5)
		self.command('mv compile_oai_enb.log ' + 'build_log_' + SSH.testCase_id, '\$', 5)
		# Workaround to run with develop-nr
		self.command('if [ -e ran_build ]; then cp -rf ran_build lte_build_oai; fi', '\$', 30)
285
		self.close()
286
		self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
287

288 289 290 291 292 293 294
	def InitializeHSS(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			logging.debug('Using the OAI EPC HSS')
295
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
296 297
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
298
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_hss 2>&1 | stdbuf -o0 awk \'{ print strftime("[%Y/%m/%d %H:%M:%S] ",systime()) $0 }\' | stdbuf -o0 tee -a hss_' + SSH.testCase_id + '.log &', 'Core state: 2 -> 3', 35)
299 300
		else:
			logging.debug('Using the ltebox simulated HSS')
301 302 303 304
			self.command('if [ -d ' + self.EPCSourceCodePath + '/scripts ]; then echo ' + self.eNBPassword + ' | sudo -S rm -Rf ' + self.EPCSourceCodePath + '/scripts ; fi', '\$', 5)
			self.command('mkdir -p ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
			self.command('cd /opt/hss_sim0609', '\$', 5)
			self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f hss.log daemon.log', '\$', 5)
305 306
			self.command('echo ' + self.EPCPassword + ' | sudo -S echo "Starting sudo session" && sudo daemon --unsafe --name=simulated_hss --chdir=/opt/hss_sim0609 ./starthss_real  ', '\$', 5)
		self.close()
307
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
308

309 310 311 312 313 314
	def InitializeMME(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
315
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
316 317
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
318
			self.command('stdbuf -o0 hostname', '\$', 5)
319 320 321 322 323 324 325
			result = re.search('hostname\\\\r\\\\n(?P<host_name>[a-zA-Z0-9\-\_]+)\\\\r\\\\n', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m Hostname Not Found! \u001B[0m')
				sys.exit(1)
			host_name = result.group('host_name')
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_mme 2>&1 | stdbuf -o0 tee -a mme_' + SSH.testCase_id + '.log &', 'MME app initialization complete', 100)
		else:
326
			self.command('cd /opt/ltebox/tools', '\$', 5)
327 328
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./start_mme', '\$', 5)
		self.close()
329
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
330 331 332

	def InitializeSPGW(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '':
333 334
			Usage()
			sys.exit('Insufficient Parameter')
335 336
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
337
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
338 339 340 341
			self.command('source oaienv', '\$', 5)
			self.command('cd scripts', '\$', 5)
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./run_spgw 2>&1 | stdbuf -o0 tee -a spgw_' + SSH.testCase_id + '.log &', 'Initializing SPGW-APP task interface: DONE', 30)
		else:
342
			self.command('cd /opt/ltebox/tools', '\$', 5)
343 344
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./start_xGw', '\$', 5)
		self.close()
345
		self.CreateHtmlTestRow(self.EPCType, 'OK', ALL_PROCESSES_OK)
346 347 348

	def InitializeeNB(self):
		if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.eNBSourceCodePath == '':
349 350 351
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = True
Raphael Defosseux's avatar
Raphael Defosseux committed
352 353 354
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.Initialize_eNB_args, 'KO', pStatus)
355
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
356
			sys.exit(1)
357 358
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
359 360
		# Initialize_eNB_args usually start with -O and followed by the location in repository
		full_config_file = self.Initialize_eNB_args.replace('-O ','')
361 362 363 364 365 366 367
		extIdx = full_config_file.find('.conf')
		if (extIdx > 0):
			extra_options = full_config_file[extIdx + 5:]
			full_config_file = full_config_file[:extIdx + 5]
			config_path, config_file = os.path.split(full_config_file)
		else:
			sys.exit('Insufficient Parameter')
368
		ci_full_config_file = config_path + '/ci-' + config_file
369 370 371 372
		rruCheck = False
		result = re.search('rru', str(config_file))
		if result is not None:
			rruCheck = True
373 374
		# Make a copy and adapt to EPC / eNB IP addresses
		self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
375 376
		self.command('sed -i -e \'s/CI_MME_IP_ADDR/' + self.EPCIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
		self.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
377
		# Launch eNB with the modified config file
378 379
		self.command('source oaienv', '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
380 381
		self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.eNBSourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(SSH.eNB_instance) + '.sh ', '\$', 5)
		self.command('chmod 775 ./my-lte-softmodem-run' + str(SSH.eNB_instance) + '.sh ', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
382
		self.command('echo ' + self.eNBPassword + ' | sudo -S rm -Rf enb_' + SSH.testCase_id + '.log', '\$', 5)
383 384 385
		self.command('echo ' + self.eNBPassword + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(SSH.eNB_instance) + '_daemon --chdir=' + self.eNBSourceCodePath + '/cmake_targets -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + SSH.testCase_id + '.log ./my-lte-softmodem-run' + str(SSH.eNB_instance) + '.sh', '\$', 5)
		if not rruCheck:
			self.eNBLogFile = 'enb_' + SSH.testCase_id + '.log'
386 387 388 389 390 391 392
		time.sleep(6)
		doLoop = True
		loopCounter = 10
		while (doLoop):
			loopCounter = loopCounter - 1
			if (loopCounter == 0):
				doLoop = False
393
				logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
394
				self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK)
395
				self.CreateHtmlFooter(False)
396 397
				self.close()
				sys.exit(1)
398
			else:
399 400 401 402 403
				self.command('stdbuf -o0 cat enb_' + SSH.testCase_id + '.log | egrep --color=never -i "wait|sync"', '\$', 4)
				if rruCheck:
					result = re.search('wait RUs', str(self.ssh.before))
				else:
					result = re.search('got sync', str(self.ssh.before))
Raphael Defosseux's avatar
Raphael Defosseux committed
404 405 406 407
				if result is None:
					time.sleep(6)
				else:
					doLoop = False
408
					self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'OK', ALL_PROCESSES_OK)
Raphael Defosseux's avatar
Raphael Defosseux committed
409
					logging.debug('\u001B[1m Initialize eNB Completed\u001B[0m')
410

411 412
		self.close()

413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
	def InitializeUE_common(self, device_id):
		logging.debug('send adb commands')
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			# The following commands are deprecated since we no longer work on Android 7+
			# self.command('stdbuf -o0 adb -s ' + device_id + ' shell settings put global airplane_mode_on 1', '\$', 10)
			# self.command('stdbuf -o0 adb -s ' + device_id + ' shell am broadcast -a android.intent.action.AIRPLANE_MODE --ez state true', '\$', 60)
			# a dedicated script has to be installed inside the UE
			# airplane mode on means call /data/local/tmp/off
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
			#airplane mode off means call /data/local/tmp/on
			logging.debug('\u001B[1mUE (' + device_id + ') Initialize Completed\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def InitializeUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		multi_jobs = []
		for device_id in self.UEDevices:
			p = Process(target = SSH.InitializeUE_common, args = (device_id,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
441
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
442

Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
443
	def AttachUE_common(self, device_id, statusQueue, lock):
444 445 446 447
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/on', '\$', 60)
			time.sleep(2)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
448 449
			max_count = 45
			count = max_count
450 451 452 453 454
			while count > 0:
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
				result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
				if result is None:
					logging.debug('\u001B[1;37;41m mDataConnectionState Not Found! \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
455 456 457 458 459 460
					lock.acquire()
					statusQueue.put(-1)
					statusQueue.put(device_id)
					statusQueue.put('mDataConnectionState Not Found!')
					lock.release()
					break
461 462 463
				mDataConnectionState = int(result.group('state'))
				if mDataConnectionState == 2:
					logging.debug('\u001B[1mUE (' + device_id + ') Attach Completed\u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
464 465 466 467 468
					lock.acquire()
					statusQueue.put(max_count - count)
					statusQueue.put(device_id)
					statusQueue.put('Attach Completed')
					lock.release()
469 470 471
					break
				count = count - 1
				if count == 15 or count == 30:
Raphael Defosseux's avatar
Raphael Defosseux committed
472
					logging.debug('\u001B[1;30;43m Retry UE (' + device_id + ') Flight Mode Off \u001B[0m')
473 474 475 476
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
					time.sleep(0.5)
					self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/on', '\$', 60)
					time.sleep(0.5)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
477
				logging.debug('\u001B[1mWait UE (' + device_id + ') a second until mDataConnectionState=2 (' + str(max_count-count) + ' times)\u001B[0m')
478 479 480
				time.sleep(1)
			if count == 0:
				logging.debug('\u001B[1;37;41m UE (' + device_id + ') Attach Failed \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
481 482 483 484 485
				lock.acquire()
				statusQueue.put(-1)
				statusQueue.put(device_id)
				statusQueue.put('Attach Failed')
				lock.release()
486 487 488 489 490 491 492 493 494
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def AttachUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
495 496 497
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
498
			self.AutoTerminateUEandeNB()
499
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
500
			sys.exit(1)
501
		multi_jobs = []
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
502 503
		status_queue = SimpleQueue()
		lock = Lock()
504
		for device_id in self.UEDevices:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
505
			p = Process(target = SSH.AttachUE_common, args = (device_id, status_queue, lock,))
506 507 508 509 510
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
511 512

		if (status_queue.empty()):
513
			self.CreateHtmlTestRow('N/A', 'KO', ALL_PROCESSES_OK)
514
			self.CreateHtmlFooter(False)
515
			self.AutoTerminateUEandeNB()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
			sys.exit(1)
		else:
			attach_status = True
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					attach_status = False
				device_id = status_queue.get()
				message = status_queue.get()
				if (count < 0):
					html_cell = "<pre>UE (" + device_id + ")\n" + message + "</pre>"
				else:
					html_cell = "<pre>UE (" + device_id + ")\n" + message + ' in ' + str(count + 2) + ' seconds</pre>'
				html_queue.put(html_cell)
			if (attach_status):
				self.CreateHtmlTestRowQueue('N/A', 'OK', len(self.UEDevices), html_queue)
			else:
				self.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
535
				self.AutoTerminateUEandeNB()
536
				self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
537
				sys.exit(1)
538 539 540 541

	def DetachUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
542
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
543 544 545 546 547 548 549 550 551 552
			logging.debug('\u001B[1mUE (' + device_id + ') Detach Completed\u001B[0m')
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def DetachUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
553 554 555
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
556
			self.AutoTerminateUEandeNB()
557
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
558
			sys.exit(1)
559 560 561 562 563 564 565 566
		multi_jobs = []
		for device_id in self.UEDevices:
			p = Process(target = SSH.DetachUE_common, args = (device_id,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
567
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612

	def RebootUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			previousmDataConnectionStates = []
			# Save mDataConnectionState
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
			result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m mDataConnectionState Not Found! \u001B[0m')
				sys.exit(1)
			previousmDataConnectionStates.append(int(result.group('state')))
			# Reboot UE
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell reboot', '\$', 10)
			time.sleep(60)
			previousmDataConnectionState = previousmDataConnectionStates.pop(0)
			count = 180
			while count > 0:
				count = count - 1
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell dumpsys telephony.registry | grep mDataConnectionState', '\$', 15)
				result = re.search('mDataConnectionState.*=(?P<state>[0-9\-]+)', str(self.ssh.before))
				if result is None:
					mDataConnectionState = None
				else:
					mDataConnectionState = int(result.group('state'))
					logging.debug('mDataConnectionState = ' + result.group('state'))
				if mDataConnectionState is None or (previousmDataConnectionState == 2 and mDataConnectionState != 2):
					logging.debug('\u001B[1mWait UE (' + device_id + ') a second until reboot completion (' + str(180-count) + ' times)\u001B[0m')
					time.sleep(1)
				else:
					logging.debug('\u001B[1mUE (' + device_id + ') Reboot Completed\u001B[0m')
					break
			if count == 0:
				logging.debug('\u001B[1;37;41m UE (' + device_id + ') Reboot Failed \u001B[0m')
				sys.exit(1)
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def RebootUE(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
613 614 615
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow('N/A', 'KO', pStatus)
616
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
617
			sys.exit(1)
618 619 620 621 622 623 624 625
		multi_jobs = []
		for device_id in self.UEDevices:
			p = Process(target = SSH.RebootUE_common, args = (device_id,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
626
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
627 628 629 630 631 632 633 634 635 636 637 638

	def GetAllUEDevices(self, terminate_ue_flag):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('adb devices', '\$', 15)
		self.UEDevices = re.findall("\\\\r\\\\n([A-Za-z0-9]+)\\\\tdevice",str(self.ssh.before))
		if terminate_ue_flag == False:
			if len(self.UEDevices) == 0:
				logging.debug('\u001B[1;37;41m UE Not Found! \u001B[0m')
				sys.exit(1)
639 640
		self.close()

641 642 643 644
	def GetAllUEIPAddresses(self):
		if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
645
		ue_ip_status = 0
646 647 648
		self.UEIPAddresses = []
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		for device_id in self.UEDevices:
649 650 651 652 653 654
			count = 0
			while count < 4:
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell ip addr show | grep rmnet', '\$', 15)
				result = re.search('inet (?P<ueipaddress>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\/[0-9]+[0-9a-zA-Z\.\s]+', str(self.ssh.before))
				if result is None:
					logging.debug('\u001B[1;37;41m UE IP Address Not Found! \u001B[0m')
655
					time.sleep(1)
656 657 658 659 660 661
					count += 1
				else:
					count = 10
			if count < 9:
				ue_ip_status -= 1
				continue
662 663 664 665 666
			UE_IPAddress = result.group('ueipaddress')
			logging.debug('\u001B[1mUE (' + device_id + ') IP Address is ' + UE_IPAddress + '\u001B[0m')
			for ueipaddress in self.UEIPAddresses:
				if ueipaddress == UE_IPAddress:
					logging.debug('\u001B[1mUE (' + device_id + ') IP Address ' + UE_IPAddress + 'has been existed!' + '\u001B[0m')
667 668
					ue_ip_status -= 1
					continue
669
			self.UEIPAddresses.append(UE_IPAddress)
670
		self.close()
671
		return ue_ip_status
672

Raphael Defosseux's avatar
Raphael Defosseux committed
673 674 675 676 677 678 679 680 681
	def ping_iperf_wrong_exit(self, lock, UE_IPAddress, device_id, statusQueue, message):
		lock.acquire()
		statusQueue.put(-1)
		statusQueue.put(device_id)
		statusQueue.put(UE_IPAddress)
		statusQueue.put(message)
		lock.release()

	def Ping_common(self, lock, UE_IPAddress, device_id, statusQueue):
682 683 684 685 686
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
			self.command('cd scripts', '\$', 5)
			ping_time = re.findall("-c (\d+)",str(self.ping_args))
Raphael Defosseux's avatar
Raphael Defosseux committed
687 688 689 690 691 692 693
			ping_status = self.command('stdbuf -o0 ping ' + self.ping_args + ' ' + UE_IPAddress + ' 2>&1 | stdbuf -o0 tee -a ping_' + SSH.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5)
			# TIMEOUT CASE
			if ping_status < 0:
				message = 'Ping with UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
				return
694 695
			result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', str(self.ssh.before))
			if result is None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
696 697
				message = 'Packet Loss Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
698
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
699
				return
700 701
			packetloss = result.group('packetloss')
			if float(packetloss) == 100:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
702 703
				message = 'Packet Loss is 100%'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
704
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
705
				return
706 707
			result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', str(self.ssh.before))
			if result is None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
708 709
				message = 'Ping RTT_Min RTT_Avg RTT_Max Not Found!'
				logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
710
				self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
711
				return
712 713 714
			rtt_min = result.group('rtt_min')
			rtt_avg = result.group('rtt_avg')
			rtt_max = result.group('rtt_max')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
715 716 717 718
			pal_msg = 'Packet Loss : ' + packetloss + '%'
			min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
			avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
			max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'
719 720
			lock.acquire()
			logging.debug('\u001B[1;37;44m ping result (' + UE_IPAddress + ') \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
721 722 723 724 725 726
			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
			qMsg = pal_msg + '\n' + min_msg + '\n' + avg_msg + '\n' + max_msg
			packetLossOK = True
727 728
			if packetloss is not None:
				if float(packetloss) > float(self.ping_packetloss_threshold):
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
729
					qMsg += '\nPacket Loss too high'
730
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
731
					packetLossOK = False
732
				elif float(packetloss) > 0:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
733
					qMsg += '\nPacket Loss is not 0%'
Raphael Defosseux's avatar
Raphael Defosseux committed
734
					logging.debug('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
735 736 737 738 739 740 741 742
			if (packetLossOK):
				statusQueue.put(0)
			else:
				statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(qMsg)
			lock.release()
743 744 745 746 747 748 749 750 751
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def Ping(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
752 753 754
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
755
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
756
			sys.exit(1)
757 758 759
		ueIpStatus = self.GetAllUEIPAddresses()
		if (ueIpStatus < 0):
			self.CreateHtmlTestRow(self.ping_args, 'KO', UE_IP_ADDRESS_ISSUE)
760
			self.CreateHtmlFooter(False)
761
			sys.exit(1)
762 763 764
		multi_jobs = []
		i = 0
		lock = Lock()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
765
		status_queue = SimpleQueue()
766 767
		for UE_IPAddress in self.UEIPAddresses:
			device_id = self.UEDevices[i]
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
768
			p = Process(target = SSH.Ping_common, args = (lock,UE_IPAddress,device_id,status_queue,))
769 770 771 772 773 774
			p.daemon = True
			p.start()
			multi_jobs.append(p)
			i = i + 1
		for job in multi_jobs:
			job.join()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
775 776

		if (status_queue.empty()):
777
			self.CreateHtmlTestRow(self.ping_args, 'KO', ALL_PROCESSES_OK)
778
			self.AutoTerminateUEandeNB()
779
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796
			sys.exit(1)
		else:
			ping_status = True
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					ping_status = False
				device_id = status_queue.get()
				ip_addr = status_queue.get()
				message = status_queue.get()
				html_cell = "<pre>UE (" + device_id + ")\nIP Address  : " + ip_addr + "\n" + message + "</pre>"
				html_queue.put(html_cell)
			if (ping_status):
				self.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue)
			else:
				self.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
797
				self.AutoTerminateUEandeNB()
798
				self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
799
				sys.exit(1)
800

801 802 803 804 805 806 807
	def Iperf_ComputeTime(self):
		result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args))
		if result is None:
			logging.debug('\u001B[1;37;41m Iperf time Not Found! \u001B[0m')
			sys.exit(1)
		return result.group('iperf_time')

808
	def Iperf_ComputeModifiedBW(self, idx, ue_num):
809 810 811 812 813
		result = re.search('-b (?P<iperf_bandwidth>[0-9\.]+)[KMG]', str(self.iperf_args))
		if result is None:
			logging.debug('\u001B[1;37;41m Iperf bandwidth Not Found! \u001B[0m')
			sys.exit(1)
		iperf_bandwidth = result.group('iperf_bandwidth')
814 815 816 817 818 819 820 821 822 823 824
		if SSH.iperf_profile == 'balanced':
			iperf_bandwidth_new = float(iperf_bandwidth)/ue_num
		if SSH.iperf_profile == 'single-ue':
			iperf_bandwidth_new = float(iperf_bandwidth)
		if SSH.iperf_profile == 'unbalanced':
			# residual is 2% of max bw
			residualBW = float(iperf_bandwidth) / 50
			if idx == 0:
				iperf_bandwidth_new = float(iperf_bandwidth) - ((ue_num - 1) * residualBW)
			else:
				iperf_bandwidth_new = residualBW
825
		iperf_bandwidth_str = '-b ' + iperf_bandwidth
826
		iperf_bandwidth_str_new = '-b ' + ('%.2f' % iperf_bandwidth_new)
827 828 829 830 831 832
		result = re.sub(iperf_bandwidth_str, iperf_bandwidth_str_new, str(self.iperf_args))
		if result is None:
			logging.debug('\u001B[1;37;41m Calculate Iperf bandwidth Failed! \u001B[0m')
			sys.exit(1)
		return result

833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858
	def Iperf_analyzeV2TCPOutput(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options):
		self.command('awk -f /tmp/tcp_iperf_stats.awk /tmp/CI-eNB/scripts/iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', 5)
		result = re.search('Avg Bitrate : (?P<average>[0-9\.]+ Mbits\/sec) Max Bitrate : (?P<maximum>[0-9\.]+ Mbits\/sec) Min Bitrate : (?P<minimum>[0-9\.]+ Mbits\/sec)', str(self.ssh.before))
		if result is not None:
			avgbitrate = result.group('average')
			maxbitrate = result.group('maximum')
			minbitrate = result.group('minimum')
			lock.acquire()
			logging.debug('\u001B[1;37;44m TCP iperf result (' + UE_IPAddress + ') \u001B[0m')
			msg = 'TCP Stats   :\n'
			if avgbitrate is not None:
				logging.debug('\u001B[1;34m    Avg Bitrate : ' + avgbitrate + '\u001B[0m')
				msg += 'Avg Bitrate : ' + avgbitrate + '\n'
			if maxbitrate is not None:
				logging.debug('\u001B[1;34m    Max Bitrate : ' + maxbitrate + '\u001B[0m')
				msg += 'Max Bitrate : ' + maxbitrate + '\n'
			if minbitrate is not None:
				logging.debug('\u001B[1;34m    Min Bitrate : ' + minbitrate + '\u001B[0m')
				msg += 'Min Bitrate : ' + minbitrate + '\n'
			statusQueue.put(0)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(msg)
			lock.release()
		return 0

859
	def Iperf_analyzeV2Output(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options):
860 861 862 863
		result = re.search('-u', str(iperf_real_options))
		if result is None:
			return self.Iperf_analyzeV2TCPOutput(lock, UE_IPAddress, device_id, statusQueue, iperf_real_options)

864 865 866 867 868 869 870
		result = re.search('Server Report:', str(self.ssh.before))
		if result is None:
			result = re.search('read failed: Connection refused', str(self.ssh.before))
			if result is not None:
				logging.debug('\u001B[1;37;41m Could not connect to iperf server! \u001B[0m')
			else:
				logging.debug('\u001B[1;37;41m Server Report and Connection refused Not Found! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
871
			return -1
872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889
		# Computing the requested bandwidth in float
		result = re.search('-b (?P<iperf_bandwidth>[0-9\.]+)[KMG]', str(iperf_real_options))
		if result is not None:
			req_bandwidth = result.group('iperf_bandwidth')
			req_bw = float(req_bandwidth)
			result = re.search('-b [0-9\.]+K', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Kbits/sec' % req_bw
				req_bw = req_bw * 1000
			result = re.search('-b [0-9\.]+M', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Mbits/sec' % req_bw
				req_bw = req_bw * 1000000
			result = re.search('-b [0-9\.]+G', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Gbits/sec' % req_bw
				req_bw = req_bw * 1000000000

890
		result = re.search('Server Report:\\\\r\\\\n(?:|\[ *\d+\].*) (?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(\d+\/..\d+) (\((?P<packetloss>[0-9\.]+)%\))', str(self.ssh.before))
891 892 893 894 895 896
		if result is not None:
			bitrate = result.group('bitrate')
			packetloss = result.group('packetloss')
			jitter = result.group('jitter')
			lock.acquire()
			logging.debug('\u001B[1;37;44m iperf result (' + UE_IPAddress + ') \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
897
			iperfStatus = True
898 899
			msg = 'Req Bitrate : ' + req_bandwidth + '\n'
			logging.debug('\u001B[1;34m    Req Bitrate : ' + req_bandwidth + '\u001B[0m')
900
			if bitrate is not None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
901
				msg += 'Bitrate     : ' + bitrate + '\n'
902
				logging.debug('\u001B[1;34m    Bitrate     : ' + bitrate + '\u001B[0m')
903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918
				result = re.search('(?P<real_bw>[0-9\.]+) [KMG]bits/sec', str(bitrate))
				if result is not None:
					actual_bw = float(str(result.group('real_bw')))
					result = re.search('[0-9\.]+ K', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000
					result = re.search('[0-9\.]+ M', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000000
					result = re.search('[0-9\.]+ G', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000000000
					br_loss = 100 * actual_bw / req_bw
					bitperf = '%.2f ' % br_loss
					msg += 'Bitrate Perf: ' + bitperf + '%\n'
					logging.debug('\u001B[1;34m    Bitrate Perf: ' + bitperf + '%\u001B[0m')
919
			if packetloss is not None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
920
				msg += 'Packet Loss : ' + packetloss + '%\n'
921 922
				logging.debug('\u001B[1;34m    Packet Loss : ' + packetloss + '%\u001B[0m')
				if float(packetloss) > float(self.iperf_packetloss_threshold):
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
923
					msg += 'Packet Loss too high!\n'
924
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
925
					iperfStatus = False
926
			if jitter is not None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
927
				msg += 'Jitter      : ' + jitter + '\n'
928
				logging.debug('\u001B[1;34m    Jitter      : ' + jitter + '\u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
929 930 931 932 933 934 935
			if (iperfStatus):
				statusQueue.put(0)
			else:
				statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			statusQueue.put(msg)
936
			lock.release()
Raphael Defosseux's avatar
Raphael Defosseux committed
937 938 939 940
			return 0

	def Iperf_analyzeV2Server(self, lock, UE_IPAddress, device_id, statusQueue, iperf_real_options):
		if (not os.path.isfile('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log')):
Raphael Defosseux's avatar
Raphael Defosseux committed
941
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log')
Raphael Defosseux's avatar
Raphael Defosseux committed
942 943 944 945 946
			return
		# Computing the requested bandwidth in float
		result = re.search('-b (?P<iperf_bandwidth>[0-9\.]+)[KMG]', str(iperf_real_options))
		if result is None:
			logging.debug('Iperf bandwidth Not Found!')
Raphael Defosseux's avatar
Raphael Defosseux committed
947
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not compute Iperf bandwidth!')
Raphael Defosseux's avatar
Raphael Defosseux committed
948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
			return
		else:
			req_bandwidth = result.group('iperf_bandwidth')
			req_bw = float(req_bandwidth)
			result = re.search('-b [0-9\.]+K', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Kbits/sec' % req_bw
				req_bw = req_bw * 1000
			result = re.search('-b [0-9\.]+M', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Mbits/sec' % req_bw
				req_bw = req_bw * 1000000
			result = re.search('-b [0-9\.]+G', str(iperf_real_options))
			if result is not None:
				req_bandwidth = '%.1f Gbits/sec' % req_bw
				req_bw = req_bw * 1000000000

		server_file = open('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log', 'r')
		br_sum = 0.0
		ji_sum = 0.0
		pl_sum = 0
		ps_sum = 0
		row_idx = 0
		for line in server_file.readlines():
			result = re.search('(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(?P<lostPack>[0-9]+)/ +(?P<sentPack>[0-9]+)', str(line))
			if result is not None:
				bitrate = result.group('bitrate')
				jitter = result.group('jitter')
				packetlost = result.group('lostPack')
				packetsent = result.group('sentPack')
				br = bitrate.split(' ')
				ji = jitter.split(' ')
				row_idx = row_idx + 1
				curr_br = float(br[0])
				pl_sum = pl_sum + int(packetlost)
				ps_sum = ps_sum + int(packetsent)
				if (br[1] == 'Kbits/sec'):
					curr_br = curr_br * 1000
				if (br[1] == 'Mbits/sec'):
					curr_br = curr_br * 1000 * 1000
				br_sum = curr_br + br_sum
				ji_sum = float(ji[0]) + ji_sum
		if (row_idx > 0):
			br_sum = br_sum / row_idx
			ji_sum = ji_sum / row_idx
			br_loss = 100 * br_sum / req_bw
			if (br_sum > 1000):
				br_sum = br_sum / 1000
				if (br_sum > 1000):
					br_sum = br_sum / 1000
					bitrate = '%.2f Mbits/sec' % br_sum
				else:
					bitrate = '%.2f Kbits/sec' % br_sum
			else:
				bitrate = '%.2f bits/sec' % br_sum
			bitperf = '%.2f ' % br_loss
			bitperf += '%'
			jitter = '%.2f ms' % (ji_sum)
			if (ps_sum > 0):
				pl = float(100 * pl_sum / ps_sum)
				packetloss = '%2.1f ' % (pl)
				packetloss += '%'
			else:
				packetloss = 'unknown'
			lock.acquire()
			if (br_loss < 90):
				statusQueue.put(1)
			else:
				statusQueue.put(0)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
			req_msg = 'Req Bitrate : ' + req_bandwidth
			bir_msg = 'Bitrate     : ' + bitrate
			brl_msg = 'Bitrate Perf: ' + bitperf
			jit_msg = 'Jitter      : ' + jitter
			pal_msg = 'Packet Loss : ' + packetloss
			statusQueue.put(req_msg + '\n' + bir_msg + '\n' + brl_msg + '\n' + jit_msg + '\n' + pal_msg + '\n')
			logging.debug('\u001B[1;37;45m iperf result (' + UE_IPAddress + ') \u001B[0m')
			logging.debug('\u001B[1;35m    ' + req_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + bir_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + brl_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + jit_msg + '\u001B[0m')
			logging.debug('\u001B[1;35m    ' + pal_msg + '\u001B[0m')
			lock.release()
		else:
Raphael Defosseux's avatar
Raphael Defosseux committed
1033
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, 'Could not analyze from server log')
Raphael Defosseux's avatar
Raphael Defosseux committed
1034 1035 1036

		server_file.close()

1037

Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1038
	def Iperf_analyzeV3Output(self, lock, UE_IPAddress, device_id, statusQueue):
1039 1040 1041
		result = re.search('(?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?:|[0-9\.]+ ms +\d+\/\d+ \((?P<packetloss>[0-9\.]+)%\)) +(?:|receiver)\\\\r\\\\n(?:|\[ *\d+\] Sent \d+ datagrams)\\\\r\\\\niperf Done\.', str(self.ssh.before))
		if result is None:
			result = re.search('(?P<error>iperf: error - [a-zA-Z0-9 :]+)', str(self.ssh.before))
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1042 1043 1044 1045
			lock.acquire()
			statusQueue.put(-1)
			statusQueue.put(device_id)
			statusQueue.put(UE_IPAddress)
1046 1047
			if result is not None:
				logging.debug('\u001B[1;37;41m ' + result.group('error') + ' \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1048
				statusQueue.put(result.group('error'))
1049 1050
			else:
				logging.debug('\u001B[1;37;41m Bitrate and/or Packet Loss Not Found! \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1051 1052 1053
				statusQueue.put('Bitrate and/or Packet Loss Not Found!')
			lock.release()

1054 1055 1056 1057 1058
		bitrate = result.group('bitrate')
		packetloss = result.group('packetloss')
		lock.acquire()
		logging.debug('\u001B[1;37;44m iperf result (' + UE_IPAddress + ') \u001B[0m')
		logging.debug('\u001B[1;34m    Bitrate     : ' + bitrate + '\u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1059 1060
		msg = 'Bitrate     : ' + bitrate + '\n'
		iperfStatus = True
1061 1062
		if packetloss is not None:
			logging.debug('\u001B[1;34m    Packet Loss : ' + packetloss + '%\u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1063
			msg += 'Packet Loss : ' + packetloss + '%\n'
1064 1065
			if float(packetloss) > float(self.iperf_packetloss_threshold):
				logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1066 1067 1068 1069 1070 1071 1072 1073
				msg += 'Packet Loss too high!\n'
				iperfStatus = False
		if (iperfStatus):
			statusQueue.put(0)
		else:
			statusQueue.put(-1)
		statusQueue.put(device_id)
		statusQueue.put(UE_IPAddress)
Raphael Defosseux's avatar
Raphael Defosseux committed
1074
		statusQueue.put(msg)
1075 1076
		lock.release()

Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1077
	def Iperf_UL_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue):
1078 1079 1080 1081
		udpIperf = True
		result = re.search('-u', str(self.iperf_args))
		if result is None:
			udpIperf = False
1082 1083 1084 1085 1086 1087 1088 1089 1090 1091
		ipnumbers = UE_IPAddress.split('.')
		if (len(ipnumbers) == 4):
			ipnumbers[3] = '1'
		EPC_Iperf_UE_IPAddress = ipnumbers[0] + '.' + ipnumbers[1] + '.' + ipnumbers[2] + '.' + ipnumbers[3]

		# Launch iperf server on EPC side
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
		self.command('rm -f iperf_server_' + SSH.testCase_id + '_' + device_id + '.log', '\$', 5)
		port = 5001 + idx
1092 1093 1094 1095
		if udpIperf:
			self.command('echo $USER; nohup iperf -u -s -i 1 -p ' + str(port) + ' > iperf_server_' + SSH.testCase_id + '_' + device_id + '.log &', self.EPCUserName, 5)
		else:
			self.command('echo $USER; nohup iperf -s -i 1 -p ' + str(port) + ' > iperf_server_' + SSH.testCase_id + '_' + device_id + '.log &', self.EPCUserName, 5)
1096 1097 1098 1099 1100 1101 1102 1103 1104
		time.sleep(0.5)
		self.close()

		# Launch iperf client on UE
		self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
		self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
		iperf_time = self.Iperf_ComputeTime()
		time.sleep(0.5)

1105 1106 1107 1108
		if udpIperf:
			modified_options = self.Iperf_ComputeModifiedBW(idx, ue_num)
		else:
			modified_options = str(self.iperf_args)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1109
		modified_options = modified_options.replace('-R','')
1110 1111 1112
		time.sleep(0.5)

		self.command('rm -f iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1113 1114 1115 1116 1117 1118 1119 1120
		iperf_status = self.command('stdbuf -o0 adb -s ' + device_id + ' shell "/data/local/tmp/iperf -c ' + EPC_Iperf_UE_IPAddress + ' ' + modified_options + ' -p ' + str(port) + '" 2>&1 | stdbuf -o0 tee -a iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0)
		# TIMEOUT Case
		if iperf_status < 0:
			self.close()
			message = 'iperf on UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT !'
			logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
			self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
			return
1121
		clientStatus = self.Iperf_analyzeV2Output(lock, UE_IPAddress, device_id, statusQueue, modified_options)
Raphael Defosseux's avatar
Raphael Defosseux committed
1122
		self.close()
1123

Raphael Defosseux's avatar
Raphael Defosseux committed
1124
		# Kill iperf server on EPC side
1125 1126 1127
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('killall --signal SIGKILL iperf', self.EPCUserName, 5)
		self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1128
		# in case of failure, retrieve server log
Raphael Defosseux's avatar
Raphael Defosseux committed
1129 1130 1131 1132
		if (clientStatus == -1):
			time.sleep(1)
			if (os.path.isfile('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log')):
				os.remove('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log')
1133
			self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, self.EPCSourceCodePath + '/scripts/iperf_server_' + SSH.testCase_id + '_' + device_id + '.log', '.')
Raphael Defosseux's avatar
Raphael Defosseux committed
1134
			self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, modified_options)
1135

Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1136
	def Iperf_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue):
1137
		try:
1138 1139 1140
			# Single-UE profile -- iperf only on one UE
			if SSH.iperf_profile == 'single-ue' and idx != 0:
				return
1141
			useIperf3 = False
1142
			udpIperf = True
1143
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1144
			# if by chance ADB server and EPC are on the same remote host, at least log collection will take care of it
Raphael Defosseux's avatar
Raphael Defosseux committed
1145 1146
			self.command('if [ ! -d ' + self.EPCSourceCodePath + '/scripts ]; then mkdir -p ' + self.EPCSourceCodePath + '/scripts ; fi', '\$', 5)
			self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
1147 1148 1149 1150 1151 1152
			# Checking if iperf / iperf3 are installed
			self.command('adb -s ' + device_id + ' shell "ls /data/local/tmp"', '\$', 5)
			result = re.search('iperf3', str(self.ssh.before))
			if result is None:
				result = re.search('iperf', str(self.ssh.before))
				if result is None:
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1153 1154
					message = 'Neither iperf nor iperf3 installed on UE!'
					logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1155
					self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1156
					return
1157 1158
			else:
				useIperf3 = True
1159 1160 1161 1162 1163
			# in case of iperf, UL has its own function
			if (not useIperf3):
				result = re.search('-R', str(self.iperf_args))
				if result is not None:
					self.close()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1164
					self.Iperf_UL_common(lock, UE_IPAddress, device_id, idx, ue_num, statusQueue)
1165 1166
					return

1167 1168 1169
			if (useIperf3):
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/iperf3 -s &', '\$', 5)
			else:
1170
				self.command('rm -f iperf_server_' + SSH.testCase_id + '_' + device_id + '.log', '\$', 5)
1171 1172 1173 1174 1175 1176
				result = re.search('-u', str(self.iperf_args))
				if result is None:
					self.command('echo $USER; nohup adb -s ' + device_id + ' shell "/data/local/tmp/iperf -s -i 1" > iperf_server_' + SSH.testCase_id + '_' + device_id + '.log &', self.ADBUserName, 5)
					udpIperf = False
				else:
					self.command('echo $USER; nohup adb -s ' + device_id + ' shell "/data/local/tmp/iperf -u -s -i 1" > iperf_server_' + SSH.testCase_id + '_' + device_id + '.log &', self.ADBUserName, 5)
1177 1178 1179 1180
			time.sleep(0.5)
			self.close()

			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
1181 1182
			self.command('cd ' + self.EPCSourceCodePath + '/scripts', '\$', 5)
			iperf_time = self.Iperf_ComputeTime()
1183 1184
			time.sleep(0.5)

1185 1186 1187 1188
			if udpIperf:
				modified_options = self.Iperf_ComputeModifiedBW(idx, ue_num)
			else:
				modified_options = str(self.iperf_args)
1189 1190
			time.sleep(0.5)

1191 1192
			self.command('rm -f iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', 5)
			if (useIperf3):
1193
				self.command('stdbuf -o0 iperf3 -c ' + UE_IPAddress + ' ' + modified_options + ' 2>&1 | stdbuf -o0 tee -a iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0)
1194

Raphael Defosseux's avatar
Raphael Defosseux committed
1195
				clientStatus = 0
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1196
				self.Iperf_analyzeV3Output(lock, UE_IPAddress, device_id, statusQueue)
1197
			else:
Raphael Defosseux's avatar
Raphael Defosseux committed
1198 1199 1200 1201 1202 1203 1204
				iperf_status = self.command('stdbuf -o0 iperf -c ' + UE_IPAddress + ' ' + modified_options + ' 2>&1 | stdbuf -o0 tee -a iperf_' + SSH.testCase_id + '_' + device_id + '.log', '\$', int(iperf_time)*5.0)
				if iperf_status < 0:
					self.close()
					message = 'iperf on UE (' + str(UE_IPAddress) + ') crashed due to TIMEOUT !'
					logging.debug('\u001B[1;37;41m ' + message + ' \u001B[0m')
					self.ping_iperf_wrong_exit(lock, UE_IPAddress, device_id, statusQueue, message)
					return
1205
				clientStatus = self.Iperf_analyzeV2Output(lock, UE_IPAddress, device_id, statusQueue, modified_options)
1206 1207 1208
			self.close()

			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
1209
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell ps | grep --color=never iperf | grep -v grep', '\$', 5)
1210 1211 1212 1213 1214
			result = re.search('shell +(?P<pid>\d+)', str(self.ssh.before))
			if result is not None:
				pid_iperf = result.group('pid')
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell kill -KILL ' + pid_iperf, '\$', 5)
			self.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1215 1216 1217 1218
			if (clientStatus == -1):
				time.sleep(1)
				if (os.path.isfile('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log')):
					os.remove('iperf_server_' + SSH.testCase_id + '_' + device_id + '.log')
1219
				self.copyin(self.ADBIPAddress, self.ADBUserName, self.ADBPassword, self.EPCSourceCodePath + '/scripts/iperf_server_' + SSH.testCase_id + '_' + device_id + '.log', '.')
Raphael Defosseux's avatar
Raphael Defosseux committed
1220
				self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, modified_options)
1221 1222 1223 1224 1225 1226 1227 1228
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def Iperf(self):
		if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
			Usage()
			sys.exit('Insufficient Parameter')
		initialize_eNB_flag = False
Raphael Defosseux's avatar
Raphael Defosseux committed
1229 1230 1231
		pStatus = self.CheckProcessExist(initialize_eNB_flag)
		if (pStatus < 0):
			self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
1232
			self.AutoTerminateUEandeNB()
1233
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
Raphael Defosseux committed
1234
			sys.exit(1)
1235 1236
		ueIpStatus = self.GetAllUEIPAddresses()
		if (ueIpStatus < 0):
Raphael Defosseux's avatar
Raphael Defosseux committed
1237
			self.CreateHtmlTestRow(self.iperf_args, 'KO', UE_IP_ADDRESS_ISSUE)
1238
			self.AutoTerminateUEandeNB()
1239
			self.CreateHtmlFooter(False)
1240
			sys.exit(1)
1241 1242 1243 1244
		multi_jobs = []
		i = 0
		ue_num = len(self.UEIPAddresses)
		lock = Lock()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1245
		status_queue = SimpleQueue()
1246 1247
		for UE_IPAddress in self.UEIPAddresses:
			device_id = self.UEDevices[i]
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1248
			p = Process(target = SSH.Iperf_common, args = (lock,UE_IPAddress,device_id,i,ue_num,status_queue,))
1249 1250 1251 1252 1253 1254
			p.daemon = True
			p.start()
			multi_jobs.append(p)
			i = i + 1
		for job in multi_jobs:
			job.join()
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1255 1256

		if (status_queue.empty()):
1257
			self.CreateHtmlTestRow(self.iperf_args, 'KO', ALL_PROCESSES_OK)
1258
			self.AutoTerminateUEandeNB()
1259
			self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1260 1261 1262
			sys.exit(1)
		else:
			iperf_status = True
Raphael Defosseux's avatar
Raphael Defosseux committed
1263
			iperf_noperf = False
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1264 1265 1266 1267 1268
			html_queue = SimpleQueue()
			while (not status_queue.empty()):
				count = status_queue.get()
				if (count < 0):
					iperf_status = False
Raphael Defosseux's avatar
Raphael Defosseux committed
1269 1270
				if (count > 0):
					iperf_noperf = True
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1271 1272 1273 1274 1275
				device_id = status_queue.get()
				ip_addr = status_queue.get()
				message = status_queue.get()
				html_cell = "<pre>UE (" + device_id + ")\nIP Address  : " + ip_addr + "\n" + message + "</pre>"
				html_queue.put(html_cell)
Raphael Defosseux's avatar
Raphael Defosseux committed
1276 1277 1278
			if (iperf_noperf and iperf_status):
				self.CreateHtmlTestRowQueue(self.iperf_args, 'PERF NOT MET', len(self.UEDevices), html_queue)
			elif (iperf_status):
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1279 1280 1281
				self.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue)
			else:
				self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
1282
				self.AutoTerminateUEandeNB()
1283
				self.CreateHtmlFooter(False)
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1284
				sys.exit(1)
1285 1286 1287

	def CheckProcessExist(self, initialize_eNB_flag):
		multi_jobs = []
Raphael Defosseux's avatar
Raphael Defosseux committed
1288 1289
		status_queue = SimpleQueue()
		p = Process(target = SSH.CheckHSSProcess, args = (status_queue,))
1290 1291 1292
		p.daemon = True
		p.start()
		multi_jobs.append(p)
Raphael Defosseux's avatar
Raphael Defosseux committed
1293
		p = Process(target = SSH.CheckMMEProcess, args = (status_queue,))
1294 1295 1296
		p.daemon = True
		p.start()
		multi_jobs.append(p)
Raphael Defosseux's avatar
Raphael Defosseux committed
1297
		p = Process(target = SSH.CheckSPGWProcess, args = (status_queue,))
1298 1299 1300 1301
		p.daemon = True
		p.start()
		multi_jobs.append(p)
		if initialize_eNB_flag == False:
Raphael Defosseux's avatar
Raphael Defosseux committed
1302
			p = Process(target = SSH.CheckeNBProcess, args = (status_queue,))
1303 1304 1305 1306 1307 1308
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()

Raphael Defosseux's avatar
Raphael Defosseux committed
1309 1310 1311 1312 1313 1314 1315 1316
		if (status_queue.empty()):
			return -15
		else:
			result = 0
			while (not status_queue.empty()):
				status = status_queue.get()
				if (status < 0):
					result = status
1317 1318 1319 1320 1321 1322 1323
			if result == ENB_PROCESS_FAILED:
				fileCheck = re.search('enb_', str(self.eNBLogFile))
				if fileCheck is not None:
					self.copyin(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, self.eNBSourceCodePath + '/cmake_targets/' + self.eNBLogFile, '.')
					logStatus = self.AnalyzeLogFile_eNB()
					if logStatus < 0:
						result = logStatus
Raphael Defosseux's avatar
Raphael Defosseux committed
1324 1325 1326
			return result

	def CheckeNBProcess(self, status_queue):
1327 1328 1329 1330 1331 1332
		try:
			self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
			self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never lte-softmodem', '\$', 5)
			result = re.search('lte-softmodem', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
1333
				status_queue.put(ENB_PROCESS_FAILED)
Raphael Defosseux's avatar
Raphael Defosseux committed
1334
			else:
1335
				status_queue.put(ENB_PROCESS_OK)
1336 1337 1338 1339
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

Raphael Defosseux's avatar
Raphael Defosseux committed
1340
	def CheckHSSProcess(self, status_queue):
1341 1342 1343 1344 1345 1346 1347 1348 1349
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never hss', '\$', 5)
			if re.match('OAI', self.EPCType, re.IGNORECASE):
				result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			else:
				result = re.search('hss_sim s6as diam_hss', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m HSS Process Not Found! \u001B[0m')
1350
				status_queue.put(HSS_PROCESS_FAILED)
Raphael Defosseux's avatar
Raphael Defosseux committed
1351
			else:
1352
				status_queue.put(HSS_PROCESS_OK)
1353 1354 1355 1356
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

Raphael Defosseux's avatar
Raphael Defosseux committed
1357
	def CheckMMEProcess(self, status_queue):
1358 1359 1360 1361 1362 1363 1364 1365 1366
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never mme', '\$', 5)
			if re.match('OAI', self.EPCType, re.IGNORECASE):
				result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			else:
				result = re.search('mme', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m MME Process Not Found! \u001B[0m')
1367
				status_queue.put(MME_PROCESS_FAILED)
Raphael Defosseux's avatar
Raphael Defosseux committed
1368
			else:
1369
				status_queue.put(MME_PROCESS_OK)
1370 1371 1372
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)
1373

Raphael Defosseux's avatar
Raphael Defosseux committed
1374
	def CheckSPGWProcess(self, status_queue):
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
		try:
			self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
			if re.match('OAI', self.EPCType, re.IGNORECASE):
				self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never spgw', '\$', 5)
				result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			else:
				self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never xGw', '\$', 5)
				result = re.search('xGw', str(self.ssh.before))
			if result is None:
				logging.debug('\u001B[1;37;41m SPGW Process Not Found! \u001B[0m')
1385
				status_queue.put(SPGW_PROCESS_FAILED)
Raphael Defosseux's avatar
Raphael Defosseux committed
1386
			else:
1387
				status_queue.put(SPGW_PROCESS_OK)
1388 1389 1390
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)
1391

1392 1393 1394 1395 1396 1397 1398 1399 1400
	def AnalyzeLogFile_eNB(self):
		if (not os.path.isfile('./' + SSH.eNBLogFile)):
			return -1
		enb_log_file = open('./' + SSH.eNBLogFile, 'r')
		foundAssertion = False
		msgAssertion = ''
		msgLine = 0
		foundSegFault = False
		foundRealTimeIssue = False
Raphael Defosseux's avatar
Raphael Defosseux committed
1401 1402 1403 1404 1405 1406 1407 1408 1409
		rrcSetupRequest = 0
		rrcSetupComplete = 0
		rrcReleaseRequest = 0
		rrcReconfigRequest = 0
		rrcReconfigComplete = 0
		rrcReestablishRequest = 0
		rrcReestablishComplete = 0
		rrcReestablishReject = 0
		uciStatMsgCount = 0
1410 1411
		pdcpFailure = 0
		ulschFailure = 0
1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
		for line in enb_log_file.readlines():
			result = re.search('[Ss]egmentation [Ff]ault', str(line))
			if result is not None:
				foundSegFault = True
			result = re.search('[Cc]ore [dD]ump', str(line))
			if result is not None:
				foundSegFault = True
			result = re.search('[Aa]ssertion', str(line))
			if result is not None:
				foundAssertion = True
			result = re.search('LLL', str(line))
			if result is not None:
				foundRealTimeIssue = True
			if foundAssertion and (msgLine < 3):
				msgLine += 1
				msgAssertion += str(line)
Raphael Defosseux's avatar
Raphael Defosseux committed
1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
			result = re.search('Generating RRCConnectionSetup', str(line))
			if result is not None:
				rrcSetupRequest += 1
			result = re.search('RRCConnectionSetupComplete from UE', str(line))
			if result is not None:
				rrcSetupComplete += 1
			result = re.search('Generate RRCConnectionRelease', str(line))
			if result is not None:
				rrcReleaseRequest += 1
			result = re.search('Generate RRCConnectionReconfiguration', str(line))
			if result is not None:
				rrcReconfigRequest += 1
			result = re.search('RRCConnectionReconfigurationComplete from UE rnti', str(line))
			if result is not None:
				rrcReconfigComplete += 1
			result = re.search('RRCConnectionReestablishmentRequest', str(line))
			if result is not None:
				rrcReestablishRequest += 1
			result = re.search('RRCConnectionReestablishmentComplete', str(line))
			if result is not None:
				rrcReestablishComplete += 1
			result = re.search('RRCConnectionReestablishmentReject', str(line))
			if result is not None:
				rrcReestablishReject += 1
			result = re.search('uci->stat', str(line))
			if result is not None:
				uciStatMsgCount += 1
1455 1456 1457 1458 1459 1460
			result = re.search('PDCP.*Out of Resources.*reason', str(line))
			if result is not None:
				pdcpFailure += 1
			result = re.search('ULSCH in error in round', str(line))
			if result is not None:
				ulschFailure += 1
1461
		enb_log_file.close()
Raphael Defosseux's avatar
Raphael Defosseux committed
1462 1463
		self.htmleNBFailureMsg = ''
		if uciStatMsgCount > 0:
1464 1465 1466 1467 1468
			statMsg = 'eNB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += statMsg + '\n'
		if pdcpFailure > 0:
			statMsg = 'eNB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
Raphael Defosseux's avatar
Raphael Defosseux committed
1469 1470
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += statMsg + '\n'
1471 1472 1473
		if ulschFailure > 0:
			statMsg = 'eNB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
1474
			self.htmleNBFailureMsg += statMsg + '\n'
Raphael Defosseux's avatar
Raphael Defosseux committed
1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502
		if rrcSetupRequest > 0 or rrcSetupComplete > 0:
			rrcMsg = 'eNB requested ' + str(rrcSetupRequest) + ' RRC Connection Setup(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
			rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
		if rrcReleaseRequest > 0:
			rrcMsg = 'eNB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
		if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
			rrcMsg = 'eNB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
			rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
		if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0:
			rrcMsg = 'eNB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
			rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
			rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
			self.htmleNBFailureMsg += rrcMsg + '\n'
1503 1504 1505 1506 1507
		if foundSegFault:
			logging.debug('\u001B[1;37;41m eNB ended with a Segmentation Fault! \u001B[0m')
			return ENB_PROCESS_SEG_FAULT
		if foundAssertion:
			logging.debug('\u001B[1;37;41m eNB ended with an assertion! \u001B[0m')
Raphael Defosseux's avatar
Raphael Defosseux committed
1508
			self.htmleNBFailureMsg += msgAssertion
1509 1510 1511 1512 1513 1514
			return ENB_PROCESS_ASSERTION
		if foundRealTimeIssue:
			logging.debug('\u001B[1;37;41m eNB faced real time issues! \u001B[0m')
			return ENB_PROCESS_REALTIME_ISSUE
		return 0

1515 1516
	def TerminateeNB(self):
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
1517
		self.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5)
1518 1519
		self.command('echo ' + self.eNBPassword + ' | sudo -S daemon --name=enb' + str(SSH.eNB_instance) + '_daemon --stop', '\$', 5)
		self.command('rm -f my-lte-softmodem-run' + str(SSH.eNB_instance) + '.sh', '\$', 5)
1520 1521
		self.command('echo ' + self.eNBPassword + ' | sudo -S killall --signal SIGINT lte-softmodem || true', '\$', 5)
		time.sleep(5)
1522
		self.command('stdbuf -o0  ps -aux | grep -v grep | grep lte-softmodem', '\$', 5)
1523 1524 1525 1526
		result = re.search('lte-softmodem', str(self.ssh.before))
		if result is not None:
			self.command('echo ' + self.eNBPassword + ' | sudo -S killall --signal SIGKILL lte-softmodem || true', '\$', 5)
		self.close()
1527 1528 1529 1530 1531 1532
		result = re.search('enb_', str(self.eNBLogFile))
		if result is not None:
			self.copyin(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, self.eNBSourceCodePath + '/cmake_targets/' + self.eNBLogFile, '.')
			logStatus = self.AnalyzeLogFile_eNB()
			if (logStatus < 0):
				self.CreateHtmlTestRow('N/A', 'KO', logStatus)
1533
				self.CreateHtmlFooter(False)
1534 1535 1536
				sys.exit(1)
			else:
				self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1537
			self.eNBLogFile = ''
1538 1539
		else:
			self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1540 1541 1542 1543 1544 1545

	def TerminateHSS(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGINT run_hss oai_hss || true', '\$', 5)
			time.sleep(2)
1546
			self.command('stdbuf -o0  ps -aux | grep -v grep | grep hss', '\$', 5)
1547 1548 1549 1550
			result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			if result is not None:
				self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL run_hss oai_hss || true', '\$', 5)
		else:
1551 1552 1553
			self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
			self.command('cd scripts', '\$', 5)
			self.command('rm -f ./kill_hss.sh', '\$', 5)
1554 1555
			self.command('echo ' + self.EPCPassword + ' | sudo -S daemon --name=simulated_hss --stop', '\$', 5)
			time.sleep(2)
1556
			self.command('ps -aux | egrep --color=never "hss_sim|simulated_hss" | grep -v grep | awk \'BEGIN{n=0}{pidId[n]=$2;n=n+1}END{print "kill -9 " pidId[0] " " pidId[1]}\' > ./kill_hss.sh', '\$', 5)
1557 1558 1559
			self.command('chmod 755 ./kill_hss.sh', '\$', 5)
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./kill_hss.sh', '\$', 5)
			self.command('rm ./kill_hss.sh', '\$', 5)
1560
		self.close()
1561
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1562 1563 1564 1565 1566 1567

	def TerminateMME(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGINT run_mme mme || true', '\$', 5)
			time.sleep(2)
1568
			self.command('stdbuf -o0 ps -aux | grep -v grep | grep mme', '\$', 5)
1569 1570 1571 1572
			result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			if result is not None:
				self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL run_mme mme || true', '\$', 5)
		else:
1573
			self.command('cd /opt/ltebox/tools', '\$', 5)
1574 1575
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./stop_mme', '\$', 5)
		self.close()
1576
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1577 1578 1579 1580 1581 1582

	def TerminateSPGW(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGINT run_spgw spgw || true', '\$', 5)
			time.sleep(2)
1583
			self.command('stdbuf -o0 ps -aux | grep -v grep | grep spgw', '\$', 5)
1584 1585 1586 1587
			result = re.search('\/bin\/bash .\/run_', str(self.ssh.before))
			if result is not None:
				self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL run_spgw spgw || true', '\$', 5)
		else:
1588
			self.command('cd /opt/ltebox/tools', '\$', 5)
1589 1590
			self.command('echo ' + self.EPCPassword + ' | sudo -S ./stop_xGw', '\$', 5)
		self.close()
1591
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1592

1593 1594 1595 1596 1597 1598
	def TerminateUE_common(self, device_id):
		try:
			self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell /data/local/tmp/off', '\$', 60)
			logging.debug('\u001B[1mUE (' + device_id + ') Detach Completed\u001B[0m')

1599 1600 1601 1602 1603
			self.command('stdbuf -o0 adb -s ' + device_id + ' shell ps | grep --color=never iperf | grep -v grep', '\$', 5)
			result = re.search('shell +(?P<pid>\d+)', str(self.ssh.before))
			if result is not None:
				pid_iperf = result.group('pid')
				self.command('stdbuf -o0 adb -s ' + device_id + ' shell kill -KILL ' + pid_iperf, '\$', 5)
1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618
			self.close()
		except:
			os.kill(os.getppid(),signal.SIGUSR1)

	def TerminateUE(self):
		terminate_ue_flag = True
		SSH.GetAllUEDevices(terminate_ue_flag)
		multi_jobs = []
		for device_id in self.UEDevices:
			p = Process(target= SSH.TerminateUE_common, args = (device_id,))
			p.daemon = True
			p.start()
			multi_jobs.append(p)
		for job in multi_jobs:
			job.join()
1619
		self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
1620

1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631
	def AutoTerminateUEandeNB(self):
		self.testCase_id = 'AUTO-KILL-UE'
		self.desc = 'Automatic Termination of UE'
		self.ShowTestID()
		self.TerminateUE()
		self.testCase_id = 'AUTO-KILL-eNB'
		self.desc = 'Automatic Termination of eNB'
		self.ShowTestID()
		self.eNB_instance = '0'
		self.TerminateeNB()

1632 1633 1634 1635
	def LogCollectBuild(self):
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1636
		self.command('rm -f build.log.zip', '\$', 5)
1637
		self.command('zip build.log.zip build_log_*/*', '\$', 60)
Raphael Defosseux's avatar
Raphael Defosseux committed
1638
		self.command('echo ' + self.eNBPassword + ' | sudo -S rm -rf build_log_*', '\$', 5)
1639
		self.close()
1640

1641 1642 1643 1644
	def LogCollecteNB(self):
		self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		self.command('cd ' + self.eNBSourceCodePath, '\$', 5)
		self.command('cd cmake_targets', '\$', 5)
1645 1646 1647
		self.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
		self.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core*', '\$', 60)
		self.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core*', '\$', 5)
1648
		self.close()
1649

1650 1651 1652 1653
	def LogCollectPing(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
		self.command('cd scripts', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1654
		self.command('rm -f ping.log.zip', '\$', 5)
1655 1656 1657
		self.command('zip ping.log.zip ping*.log', '\$', 60)
		self.command('rm ping*.log', '\$', 5)
		self.close()
1658

1659 1660 1661 1662
	def LogCollectIperf(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
		self.command('cd scripts', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1663
		self.command('rm -f iperf.log.zip', '\$', 5)
1664 1665 1666
		self.command('zip iperf.log.zip iperf*.log', '\$', 60)
		self.command('rm iperf*.log', '\$', 5)
		self.close()
1667

1668 1669
	def LogCollectHSS(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
1670 1671
		self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
		self.command('cd scripts', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1672
		self.command('rm -f hss.log.zip', '\$', 5)
1673 1674 1675
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('zip hss.log.zip hss*.log', '\$', 60)
			self.command('rm hss*.log', '\$', 5)
1676 1677 1678
		else:
			self.command('cp /opt/hss_sim0609/hss.log .', '\$', 60)
			self.command('zip hss.log.zip hss.log', '\$', 60)
1679
		self.close()
1680

1681 1682 1683
	def LogCollectMME(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
1684
		self.command('cd scripts', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1685
		self.command('rm -f mme.log.zip', '\$', 5)
1686 1687 1688 1689
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('zip mme.log.zip mme*.log', '\$', 60)
			self.command('rm mme*.log', '\$', 5)
		else:
1690 1691
			self.command('cp /opt/ltebox/var/log/*Log.0 .', '\$', 5)
			self.command('zip mme.log.zip mmeLog.0 s1apcLog.0 s1apsLog.0 s11cLog.0 libLog.0 s1apCodecLog.0', '\$', 60)
1692
		self.close()
1693

1694 1695 1696
	def LogCollectSPGW(self):
		self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword)
		self.command('cd ' + self.EPCSourceCodePath, '\$', 5)
1697
		self.command('cd scripts', '\$', 5)
Raphael Defosseux's avatar
Raphael Defosseux committed
1698
		self.command('rm -f spgw.log.zip', '\$', 5)
1699 1700 1701 1702
		if re.match('OAI', self.EPCType, re.IGNORECASE):
			self.command('zip spgw.log.zip spgw*.log', '\$', 60)
			self.command('rm spgw*.log', '\$', 5)
		else:
1703 1704
			self.command('cp /opt/ltebox/var/log/xGwLog.0 .', '\$', 5)
			self.command('zip spgw.log.zip xGwLog.0', '\$', 60)
1705 1706
		self.close()
#-----------------------------------------------------------
Raphael Defosseux's avatar
Raphael Defosseux committed
1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721
# HTML Reporting....
#-----------------------------------------------------------
	def CreateHtmlHeader(self):
		if (not self.htmlHeaderCreated):
			self.htmlFile = open('test_results.html', 'w')
			self.htmlFile.write('<!DOCTYPE html>\n')
			self.htmlFile.write('<html class="no-js" lang="en-US">\n')
			self.htmlFile.write('<head>\n')
			self.htmlFile.write('  <title>Test Results for TEMPLATE_JOB_NAME job build #TEMPLATE_BUILD_ID</title>\n')
			self.htmlFile.write('</head>\n')
			self.htmlFile.write('<body>\n')
			self.htmlFile.write('  <table style="border-collapse: collapse; border: none;">\n')
			self.htmlFile.write('    <tr style="border-collapse: collapse; border: none;">\n')
			self.htmlFile.write('      <td style="border-collapse: collapse; border: none;">\n')
			self.htmlFile.write('        <a href="http://www.openairinterface.org/">\n')
1722
			self.htmlFile.write('           <img src="http://www.openairinterface.org/wp-content/uploads/2016/03/cropped-oai_final_logo2.png" alt="" border="none" height=50 width=150>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
			self.htmlFile.write('           </img>\n')
			self.htmlFile.write('        </a>\n')
			self.htmlFile.write('      </td>\n')
			self.htmlFile.write('      <td style="border-collapse: collapse; border: none; vertical-align: center;">\n')
			self.htmlFile.write('        <b><font size = "6">Job Summary -- Job: TEMPLATE_JOB_NAME -- Build-ID: TEMPLATE_BUILD_ID</font></b>\n')
			self.htmlFile.write('      </td>\n')
			self.htmlFile.write('    </tr>\n')
			self.htmlFile.write('  </table>\n')
			self.htmlFile.write('  <br>\n')
			self.htmlFile.write('  <table border = "1">\n')
			self.htmlFile.write('     <tr>\n')
1734 1735 1736 1737
			self.htmlFile.write('       <td bgcolor = "lightcyan" >Build Start Time (UTC)</td>\n')
			self.htmlFile.write('       <td>TEMPLATE_BUILD_TIME</td>\n')
			self.htmlFile.write('     </tr>\n')
			self.htmlFile.write('     <tr>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1738
			self.htmlFile.write('       <td bgcolor = "lightcyan" >GIT Repository</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1739
			self.htmlFile.write('       <td><a href="' + SSH.eNBRepository + '">' + SSH.eNBRepository + '</a></td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764
			self.htmlFile.write('     </tr>\n')
			self.htmlFile.write('     <tr>\n')
			self.htmlFile.write('       <td bgcolor = "lightcyan" >Job Trigger</td>\n')
			if (SSH.eNB_AllowMerge):
				self.htmlFile.write('       <td>Merge-Request</td>\n')
			else:
				self.htmlFile.write('       <td>Push to Branch</td>\n')
			self.htmlFile.write('     </tr>\n')
			self.htmlFile.write('     <tr>\n')
			if (SSH.eNB_AllowMerge):
				self.htmlFile.write('       <td bgcolor = "lightcyan" >Source Branch</td>\n')
			else:
				self.htmlFile.write('       <td bgcolor = "lightcyan" >Branch</td>\n')
			self.htmlFile.write('       <td>' + SSH.eNBBranch + '</td>\n')
			self.htmlFile.write('     </tr>\n')
			self.htmlFile.write('     <tr>\n')
			if (SSH.eNB_AllowMerge):
				self.htmlFile.write('       <td bgcolor = "lightcyan" >Source Commit ID</td>\n')
			else:
				self.htmlFile.write('       <td bgcolor = "lightcyan" >Commit ID</td>\n')
			self.htmlFile.write('       <td>' + SSH.eNBCommitID + '</td>\n')
			self.htmlFile.write('     </tr>\n')
			if (SSH.eNB_AllowMerge):
				self.htmlFile.write('     <tr>\n')
				self.htmlFile.write('       <td bgcolor = "lightcyan" >Target Branch</td>\n')
1765 1766 1767 1768
				if (self.eNBTargetBranch == ''):
					self.htmlFile.write('       <td>develop</td>\n')
				else:
					self.htmlFile.write('       <td>' + self.eNBTargetBranch + '</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1769 1770 1771 1772 1773 1774 1775
				self.htmlFile.write('     </tr>\n')
			self.htmlFile.write('  </table>\n')

			terminate_ue_flag = True
			SSH.GetAllUEDevices(terminate_ue_flag)
			self.htmlUEConnected = len(self.UEDevices)

1776
			self.htmlFile.write('  <h2><a href="#FinalStatus">Jump to Final Status</a></h2>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1777
			self.htmlFile.write('  <br>\n')
1778 1779 1780 1781

			self.htmlFile.write('  <h2>' + str(self.htmlUEConnected) + ' UE(s) is(are) connected to ADB bench server</h2>\n')
			self.htmlFile.write('  <br>\n')

Raphael Defosseux's avatar
Raphael Defosseux committed
1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795
			self.htmlFile.write('  <h2>Test Summary for ' + SSH.testXMLfile + '</h2>\n')
			self.htmlFile.write('  <table border = "1">\n')
			self.htmlFile.write('      <tr bgcolor = "#33CCFF" >\n')
			self.htmlFile.write('        <th>Test Id</th>\n')
			self.htmlFile.write('        <th>Test Desc</th>\n')
			self.htmlFile.write('        <th>Test Options</th>\n')
			self.htmlFile.write('        <th>Test Status</th>\n')
			i = 0
			while (i < self.htmlUEConnected):
				self.htmlFile.write('        <th>UE' + str(i) + ' Status</th>\n')
				i += 1
			self.htmlFile.write('      </tr>\n')
		self.htmlHeaderCreated = True

1796
	def CreateHtmlFooter(self, passStatus):
Raphael Defosseux's avatar
Raphael Defosseux committed
1797
		if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)):
1798 1799 1800 1801 1802 1803 1804
			self.htmlFile.write('      <tr id="FinalStatus">\n')
			self.htmlFile.write('        <th bgcolor = "#33CCFF" colspan=2>Final Status</th>\n')
			if passStatus:
				self.htmlFile.write('        <th bgcolor = "green" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">PASS</font></th>\n')
			else:
				self.htmlFile.write('        <th bgcolor = "red" colspan=' + str(2 + self.htmlUEConnected) + '><font color="white">FAIL</font></th>\n')
			self.htmlFile.write('      </tr>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1805 1806 1807 1808 1809 1810
			self.htmlFile.write('  </table>\n')
			self.htmlFile.write('</body>\n')
			self.htmlFile.write('</html>\n')
			self.htmlFile.close()
		self.htmlFooterCreated = False

Raphael Defosseux's avatar
Raphael Defosseux committed
1811
	def CreateHtmlTestRow(self, options, status, processesStatus):
Raphael Defosseux's avatar
Raphael Defosseux committed
1812 1813 1814 1815 1816 1817 1818 1819
		if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)):
			self.htmlFile.write('      <tr>\n')
			self.htmlFile.write('        <td bgcolor = "lightcyan" >' + SSH.testCase_id  + '</td>\n')
			self.htmlFile.write('        <td>' + SSH.desc  + '</td>\n')
			self.htmlFile.write('        <td>' + str(options)  + '</td>\n')
			if (str(status) == 'OK'):
				self.htmlFile.write('        <td bgcolor = "lightgreen" >' + str(status)  + '</td>\n')
			elif (str(status) == 'KO'):
Raphael Defosseux's avatar
Raphael Defosseux committed
1820 1821
				if (processesStatus == 0):
					self.htmlFile.write('        <td bgcolor = "lightcoral" >' + str(status)  + '</td>\n')
1822
				elif (processesStatus == ENB_PROCESS_FAILED):
Raphael Defosseux's avatar
Raphael Defosseux committed
1823
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - eNB process not found</td>\n')
1824 1825 1826 1827 1828 1829 1830
				elif (processesStatus == ENB_PROCESS_SEG_FAULT):
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - eNB process ended in Segmentation Fault</td>\n')
				elif (processesStatus == ENB_PROCESS_ASSERTION):
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - eNB process ended in Assertion</td>\n')
				elif (processesStatus == ENB_PROCESS_REALTIME_ISSUE):
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - eNB process faced Real Time issue(s)/td>\n')
				elif (processesStatus == HSS_PROCESS_FAILED):
Raphael Defosseux's avatar
Raphael Defosseux committed
1831
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - HSS process not found</td>\n')
1832
				elif (processesStatus == MME_PROCESS_FAILED):
Raphael Defosseux's avatar
Raphael Defosseux committed
1833
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - MME process not found</td>\n')
1834
				elif (processesStatus == SPGW_PROCESS_FAILED):
Raphael Defosseux's avatar
Raphael Defosseux committed
1835
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - SPGW process not found</td>\n')
1836 1837
				elif (processesStatus == UE_IP_ADDRESS_ISSUE):
					self.htmlFile.write('        <td bgcolor = "lightcoral" >KO - Could not retrieve UE IP address</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1838 1839
				else:
					self.htmlFile.write('        <td bgcolor = "lightcoral" >' + str(status)  + '</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1840 1841
			else:
				self.htmlFile.write('        <td bgcolor = "orange" >' + str(status)  + '</td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1842
			if (len(str(self.htmleNBFailureMsg)) > 2):
1843 1844 1845 1846 1847 1848 1849 1850 1851
				cellBgColor = 'white'
				result = re.search('ended with|faced real time issues', self.htmleNBFailureMsg)
				if result is not None:
					cellBgColor = 'red'
				else:
					result = re.search('showed|Reestablishment', self.htmleNBFailureMsg)
					if result is not None:
						cellBgColor = 'orange'
				self.htmlFile.write('        <td bgcolor = "' + cellBgColor + '" colspan=' + str(self.htmlUEConnected) + '><pre>' + self.htmleNBFailureMsg + '</pre></td>\n')
Raphael Defosseux's avatar
Raphael Defosseux committed
1852
				self.htmleNBFailureMsg = ''
1853 1854 1855 1856 1857
			else:
				i = 0
				while (i < self.htmlUEConnected):
					self.htmlFile.write('        <td>-</td>\n')
					i += 1
Raphael Defosseux's avatar
Raphael Defosseux committed
1858 1859
			self.htmlFile.write('      </tr>\n')

Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1860 1861
	def CreateHtmlTestRowQueue(self, options, status, ue_status, ue_queue):
		if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)):
Raphael Defosseux's avatar
Raphael Defosseux committed
1862
			addOrangeBK = False
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1863 1864 1865 1866 1867 1868 1869 1870 1871
			self.htmlFile.write('      <tr>\n')
			self.htmlFile.write('        <td bgcolor = "lightcyan" >' + SSH.testCase_id  + '</td>\n')
			self.htmlFile.write('        <td>' + SSH.desc  + '</td>\n')
			self.htmlFile.write('        <td>' + str(options)  + '</td>\n')
			if (str(status) == 'OK'):
				self.htmlFile.write('        <td bgcolor = "lightgreen" >' + str(status)  + '</td>\n')
			elif (str(status) == 'KO'):
				self.htmlFile.write('        <td bgcolor = "lightcoral" >' + str(status)  + '</td>\n')
			else:
Raphael Defosseux's avatar
Raphael Defosseux committed
1872
				addOrangeBK = True
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1873 1874 1875 1876 1877
				self.htmlFile.write('        <td bgcolor = "orange" >' + str(status)  + '</td>\n')
			i = 0
			while (i < self.htmlUEConnected):
				if (i < ue_status):
					if (not ue_queue.empty()):
Raphael Defosseux's avatar
Raphael Defosseux committed
1878 1879 1880 1881
						if (addOrangeBK):
							self.htmlFile.write('        <td bgcolor = "orange" >' + str(ue_queue.get()) + '</td>\n')
						else:
							self.htmlFile.write('        <td>' + str(ue_queue.get()) + '</td>\n')
Raphael Defosseux's avatar
CI:  
Raphael Defosseux committed
1882 1883 1884 1885 1886 1887 1888
					else:
						self.htmlFile.write('        <td>-</td>\n')
				else:
					self.htmlFile.write('        <td>-</td>\n')
				i += 1
			self.htmlFile.write('      </tr>\n')

1889 1890 1891 1892 1893 1894 1895 1896 1897
#-----------------------------------------------------------
# ShowTestID()
#-----------------------------------------------------------
	def ShowTestID(self):
		logging.debug('\u001B[1m----------------------------------------\u001B[0m')
		logging.debug('\u001B[1mTest ID:' + self.testCase_id + '\u001B[0m')
		logging.debug('\u001B[1m' + self.desc + '\u001B[0m')
		logging.debug('\u001B[1m----------------------------------------\u001B[0m')

Raphael Defosseux's avatar
Raphael Defosseux committed
1898
#-----------------------------------------------------------
1899 1900 1901 1902 1903 1904 1905 1906 1907 1908
# Usage()
#-----------------------------------------------------------
def Usage():
	print('------------------------------------------------------------')
	print('main.py Ver:' + Version)
	print('------------------------------------------------------------')
	print('Usage: python main.py [options]')
	print('  --help  Show this help.')
	print('  --mode=[Mode]')
	print('      TesteNB')
1909 1910
	print('      TerminateeNB, TerminateUE, TerminateHSS, TerminateMME, TerminateSPGW')
	print('      LogCollectBuild, LogCollecteNB, LogCollectHSS, LogCollectMME, LogCollectSPGW, LogCollectPing, LogCollectIperf')
1911 1912 1913 1914
	print('  --eNBIPAddress=[eNB\'s IP Address]')
	print('  --eNBRepository=[eNB\'s Repository URL]')
	print('  --eNBBranch=[eNB\'s Branch Name]')
	print('  --eNBCommitID=[eNB\'s Commit Number]')
1915 1916
	print('  --eNB_AllowMerge=[eNB\'s Allow Merge Request (with target branch)]')
	print('  --eNBTargetBranch=[eNB\'s Target Branch in case of a Merge Request]')
1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927
	print('  --eNBUserName=[eNB\'s Login User Name]')
	print('  --eNBPassword=[eNB\'s Login Password]')
	print('  --eNBSourceCodePath=[eNB\'s Source Code Path]')
	print('  --EPCIPAddress=[EPC\'s IP Address]')
	print('  --EPCUserName=[EPC\'s Login User Name]')
	print('  --EPCPassword=[EPC\'s Login Password]')
	print('  --EPCSourceCodePath=[EPC\'s Source Code Path]')
	print('  --EPCType=[EPC\'s Type: OAI or ltebox]')
	print('  --ADBIPAddress=[ADB\'s IP Address]')
	print('  --ADBUserName=[ADB\'s Login User Name]')
	print('  --ADBPassword=[ADB\'s Login Password]')
1928
	print('  --XMLTestFile=[XML Test File to be run]')
1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939
	print('------------------------------------------------------------')

def CheckClassValidity(action,id):
	if action != 'Build_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW':
		logging.debug('ERROR: test-case ' + id + ' has wrong class ' + action)
		return False
	return True

def GetParametersFromXML(action):
	if action == 'Build_eNB':
		SSH.Build_eNB_args = test.findtext('Build_eNB_args')
1940

1941 1942
	if action == 'Initialize_eNB':
		SSH.Initialize_eNB_args = test.findtext('Initialize_eNB_args')
1943 1944 1945 1946 1947 1948 1949 1950
		SSH.eNB_instance = test.findtext('eNB_instance')
		if (SSH.eNB_instance is None):
			SSH.eNB_instance = '0'

	if action == 'Terminate_eNB':
		SSH.eNB_instance = test.findtext('eNB_instance')
		if (SSH.eNB_instance is None):
			SSH.eNB_instance = '0'
1951

1952 1953 1954 1955 1956 1957 1958
	if action == 'Ping':
		SSH.ping_args = test.findtext('ping_args')
		SSH.ping_packetloss_threshold = test.findtext('ping_packetloss_threshold')

	if action == 'Iperf':
		SSH.iperf_args = test.findtext('iperf_args')
		SSH.iperf_packetloss_threshold = test.findtext('iperf_packetloss_threshold')
1959 1960 1961 1962 1963 1964 1965
		SSH.iperf_profile = test.findtext('iperf_profile')
		if (SSH.iperf_profile is None):
			SSH.iperf_profile = 'balanced'
		else:
			if SSH.iperf_profile != 'balanced' and SSH.iperf_profile != 'unbalanced' and SSH.iperf_profile != 'single-ue':
				logging.debug('ERROR: test-case has wrong profile ' + SSH.iperf_profile)
				SSH.iperf_profile = 'balanced'
1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001

#check if given test is in list
#it is in list if one of the strings in 'list' is at the beginning of 'test'
def test_in_list(test, list):
	for check in list:
		check=check.replace('+','')
		if (test.startswith(check)):
			return True
	return False

def receive_signal(signum, frame):
	sys.exit(1)

#-----------------------------------------------------------
# Parameter Check
#-----------------------------------------------------------
mode = ''
SSH = SSHConnection()

argvs = sys.argv
argc = len(argvs)

while len(argvs) > 1:
	myArgv = argvs.pop(1)	# 0th is this file's name
	if re.match('^\-\-help$', myArgv, re.IGNORECASE):
		Usage()
		sys.exit(0)
	elif re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE)
		mode = matchReg.group(1)
	elif re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE)
		SSH.eNBIPAddress = matchReg.group(1)
	elif re.match('^\-\-eNBRepository=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-eNBRepository=(.+)$', myArgv, re.IGNORECASE)
		SSH.eNBRepository = matchReg.group(1)
Raphael Defosseux's avatar
Raphael Defosseux committed
2002 2003 2004 2005 2006
	elif re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE)
		doMerge = matchReg.group(1)
		if ((doMerge == 'true') or (doMerge == 'True')):
			SSH.eNB_AllowMerge = True
2007 2008 2009 2010 2011 2012
	elif re.match('^\-\-eNBBranch=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-eNBBranch=(.+)$', myArgv, re.IGNORECASE)
		SSH.eNBBranch = matchReg.group(1)
	elif re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE)
		SSH.eNBCommitID = matchReg.group(1)
2013 2014 2015
	elif re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE)
		SSH.eNBTargetBranch = matchReg.group(1)
2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054
	elif re.match('^\-\-eNBUserName=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-eNBUserName=(.+)$', myArgv, re.IGNORECASE)
		SSH.eNBUserName = matchReg.group(1)
	elif re.match('^\-\-eNBPassword=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-eNBPassword=(.+)$', myArgv, re.IGNORECASE)
		SSH.eNBPassword = matchReg.group(1)
	elif re.match('^\-\-eNBSourceCodePath=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-eNBSourceCodePath=(.+)$', myArgv, re.IGNORECASE)
		SSH.eNBSourceCodePath = matchReg.group(1)
	elif re.match('^\-\-EPCIPAddress=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCIPAddress=(.+)$', myArgv, re.IGNORECASE)
		SSH.EPCIPAddress = matchReg.group(1)
	elif re.match('^\-\-EPCBranch=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCBranch=(.+)$', myArgv, re.IGNORECASE)
		SSH.EPCBranch = matchReg.group(1)
	elif re.match('^\-\-EPCUserName=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCUserName=(.+)$', myArgv, re.IGNORECASE)
		SSH.EPCUserName = matchReg.group(1)
	elif re.match('^\-\-EPCPassword=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCPassword=(.+)$', myArgv, re.IGNORECASE)
		SSH.EPCPassword = matchReg.group(1)
	elif re.match('^\-\-EPCSourceCodePath=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCSourceCodePath=(.+)$', myArgv, re.IGNORECASE)
		SSH.EPCSourceCodePath = matchReg.group(1)
	elif re.match('^\-\-EPCType=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-EPCType=(.+)$', myArgv, re.IGNORECASE)
		if re.match('OAI', matchReg.group(1), re.IGNORECASE) or re.match('ltebox', matchReg.group(1), re.IGNORECASE):
			SSH.EPCType = matchReg.group(1)
		else:
			sys.exit('Invalid EPC Type: ' + matchReg.group(1) + ' -- (should be OAI or ltebox)')
	elif re.match('^\-\-ADBIPAddress=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-ADBIPAddress=(.+)$', myArgv, re.IGNORECASE)
		SSH.ADBIPAddress = matchReg.group(1)
	elif re.match('^\-\-ADBUserName=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-ADBUserName=(.+)$', myArgv, re.IGNORECASE)
		SSH.ADBUserName = matchReg.group(1)
	elif re.match('^\-\-ADBPassword=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-ADBPassword=(.+)$', myArgv, re.IGNORECASE)
		SSH.ADBPassword = matchReg.group(1)
2055 2056 2057
	elif re.match('^\-\-XMLTestFile=(.+)$', myArgv, re.IGNORECASE):
		matchReg = re.match('^\-\-XMLTestFile=(.+)$', myArgv, re.IGNORECASE)
		SSH.testXMLfile = matchReg.group(1)
2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073
	else:
		Usage()
		sys.exit('Invalid Parameter: ' + myArgv)

if re.match('^TerminateeNB$', mode, re.IGNORECASE):
	if SSH.eNBIPAddress == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '':
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.TerminateeNB()
elif re.match('^TerminateUE$', mode, re.IGNORECASE):
	if SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '':
		Usage()
		sys.exit('Insufficient Parameter')
	signal.signal(signal.SIGUSR1, receive_signal)
	SSH.TerminateUE()
elif re.match('^TerminateHSS$', mode, re.IGNORECASE):
2074
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
2075 2076 2077 2078
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.TerminateHSS()
elif re.match('^TerminateMME$', mode, re.IGNORECASE):
2079
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
2080 2081 2082 2083
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.TerminateMME()
elif re.match('^TerminateSPGW$', mode, re.IGNORECASE):
2084
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.TerminateSPGW()
elif re.match('^LogCollectBuild$', mode, re.IGNORECASE):
	if SSH.eNBIPAddress == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '':
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectBuild()
elif re.match('^LogCollecteNB$', mode, re.IGNORECASE):
	if SSH.eNBIPAddress == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '':
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollecteNB()
elif re.match('^LogCollectHSS$', mode, re.IGNORECASE):
2099
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
2100 2101 2102 2103
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectHSS()
elif re.match('^LogCollectMME$', mode, re.IGNORECASE):
2104
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
2105 2106 2107 2108
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectMME()
elif re.match('^LogCollectSPGW$', mode, re.IGNORECASE):
2109
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '':
2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectSPGW()
elif re.match('^LogCollectPing$', mode, re.IGNORECASE):
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCSourceCodePath == '':
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectPing()
elif re.match('^LogCollectIperf$', mode, re.IGNORECASE):
	if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCSourceCodePath == '':
		Usage()
		sys.exit('Insufficient Parameter')
	SSH.LogCollectIperf()
elif re.match('^TesteNB$', mode, re.IGNORECASE):
	if SSH.eNBIPAddress == '' or SSH.eNBRepository == '' or SSH.eNBBranch == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '' or SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '' or SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '':
		Usage()
		sys.exit('Insufficient Parameter')
2127

2128
	SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/tcp_iperf_stats.awk", "/tmp")
Raphael Defosseux's avatar
Raphael Defosseux committed
2129 2130
	SSH.CreateHtmlHeader()

2131
	#read test_case_list.xml file
2132 2133 2134 2135 2136
        # if no parameters for XML file, use default value
	if SSH.testXMLfile == '':
		xml_test_file = sys.path[0] + "/test_case_list.xml"
	else:
		xml_test_file = sys.path[0] + "/" + SSH.testXMLfile
2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174

	xmlTree = ET.parse(xml_test_file)
	xmlRoot = xmlTree.getroot()

	exclusion_tests=xmlRoot.findtext('TestCaseExclusionList',default='')
	requested_tests=xmlRoot.findtext('TestCaseRequestedList',default='')
	all_tests=xmlRoot.findall('testCase')

	exclusion_tests=exclusion_tests.split()
	requested_tests=requested_tests.split()

	#check that exclusion tests are well formatted
	#(6 digits or less than 6 digits followed by +)
	for test in exclusion_tests:
		if     (not re.match('^[0-9]{6}$', test) and
				not re.match('^[0-9]{1,5}\+$', test)):
			logging.debug('ERROR: exclusion test is invalidly formatted: ' + test)
			sys.exit(1)
		else:
			logging.debug(test)

	#check that requested tests are well formatted
	#(6 digits or less than 6 digits followed by +)
	#be verbose
	for test in requested_tests:
		if     (re.match('^[0-9]{6}$', test) or
				re.match('^[0-9]{1,5}\+$', test)):
			logging.debug('INFO: test group/case requested: ' + test)
		else:
			logging.debug('ERROR: requested test is invalidly formatted: ' + test)
			sys.exit(1)

	#get the list of tests to be done
	todo_tests=[]
	for test in requested_tests:
		if    (test_in_list(test, exclusion_tests)):
			logging.debug('INFO: test will be skipped: ' + test)
		else:
2175
			#logging.debug('INFO: test will be run: ' + test)
2176
			todo_tests.append(test)
2177

2178
	signal.signal(signal.SIGUSR1, receive_signal)
2179

2180 2181 2182 2183 2184 2185 2186 2187 2188 2189
	for test_case_id in todo_tests:
		for test in all_tests:
			id = test.get('id')
			if test_case_id != id:
				continue
			SSH.testCase_id = id
			SSH.desc = test.findtext('desc')
			action = test.findtext('class')
			if (CheckClassValidity(action, id) == False):
				continue
2190
			SSH.ShowTestID()
2191 2192
			GetParametersFromXML(action)
			if action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Ping' or action == 'Iperf' or action == 'Reboot_UE':
2193 2194
				terminate_ue_flag = False
				SSH.GetAllUEDevices(terminate_ue_flag)
2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228
			if action == 'Build_eNB':
				SSH.BuildeNB()
			elif action == 'Initialize_eNB':
				SSH.InitializeeNB()
			elif action == 'Terminate_eNB':
				SSH.TerminateeNB()
			elif action == 'Initialize_UE':
				SSH.InitializeUE()
			elif action == 'Terminate_UE':
				SSH.TerminateUE()
			elif action == 'Attach_UE':
				SSH.AttachUE()
			elif action == 'Detach_UE':
				SSH.DetachUE()
			elif action == 'Ping':
				SSH.Ping()
			elif action == 'Iperf':
				SSH.Iperf()
			elif action == 'Reboot_UE':
				SSH.RebootUE()
			elif action == 'Initialize_HSS':
				SSH.InitializeHSS()
			elif action == 'Terminate_HSS':
				SSH.TerminateHSS()
			elif action == 'Initialize_MME':
				SSH.InitializeMME()
			elif action == 'Terminate_MME':
				SSH.TerminateMME()
			elif action == 'Initialize_SPGW':
				SSH.InitializeSPGW()
			elif action == 'Terminate_SPGW':
				SSH.TerminateSPGW()
			else:
				sys.exit('Invalid action')
Raphael Defosseux's avatar
Raphael Defosseux committed
2229

2230
	SSH.CreateHtmlFooter(True)
2231 2232 2233 2234
else:
	Usage()
	sys.exit('Invalid mode')
sys.exit(0)