cls_oaicitest.py 64.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
#---------------------------------------------------------------------
# 
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------


#-----------------------------------------------------------
# Import Libs
#-----------------------------------------------------------
import sys		# arg
import re		# reg
import pexpect	# pexpect
import time		# sleep
import os
import subprocess
import xml.etree.ElementTree as ET
import logging
import datetime
import signal
Remi Hardy's avatar
Remi Hardy committed
45
import statistics as stat
46
from multiprocessing import SimpleQueue, Lock
47
import concurrent.futures
Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
48
import json
49

Remi Hardy's avatar
Remi Hardy committed
50 51 52
#import our libs
import helpreadme as HELP
import constants as CONST
53
import cls_cluster as OC
hardy's avatar
hardy committed
54
import sshconnection
55

56
import cls_module
57
import cls_cmd
Remi Hardy's avatar
Remi Hardy committed
58

hardy's avatar
hardy committed
59
logging.getLogger("matplotlib").setLevel(logging.WARNING)
hardy's avatar
hardy committed
60 61
import matplotlib.pyplot as plt
import numpy as np
Remi Hardy's avatar
Remi Hardy committed
62

63 64 65 66 67 68
#-----------------------------------------------------------
# Helper functions used here and in other classes
#-----------------------------------------------------------
def Iperf_ComputeModifiedBW(idx, ue_num, profile, args):
	result = re.search('-b\s*(?P<iperf_bandwidth>[0-9\.]+)(?P<unit>[KMG])', str(args))
	if result is None:
69 70 71 72
		raise ValueError(f'requested iperf bandwidth not found in iperf options "{args}"')
	iperf_bandwidth = float(result.group('iperf_bandwidth'))
	if iperf_bandwidth == 0:
		raise ValueError('iperf bandwidth set to 0 - invalid value')
73
	if profile == 'balanced':
74 75 76
		iperf_bandwidth_new = iperf_bandwidth/ue_num
	if profile =='single-ue':
		iperf_bandwidth_new = iperf_bandwidth
77 78
	if profile == 'unbalanced':
		# residual is 2% of max bw
79
		residualBW = iperf_bandwidth / 50
80
		if idx == 0:
81
			iperf_bandwidth_new = iperf_bandwidth - ((ue_num - 1) * residualBW)
82 83 84 85 86
		else:
			iperf_bandwidth_new = residualBW
	iperf_bandwidth_str = result.group(0)
	iperf_bandwidth_unit = result.group(2)
	iperf_bandwidth_str_new = f"-b {'%.2f' % iperf_bandwidth_new}{iperf_bandwidth_unit}"
87 88 89 90
	args_new = re.sub(iperf_bandwidth_str, iperf_bandwidth_str_new, str(args))
	if iperf_bandwidth_unit == 'K':
		iperf_bandwidth_new = iperf_bandwidth_new / 1000
	return iperf_bandwidth_new, args_new
91 92 93 94 95 96 97

def Iperf_ComputeTime(args):
	result = re.search('-t\s*(?P<iperf_time>\d+)', str(args))
	if result is None:
		raise Exception('Iperf time not found!')
	return int(result.group('iperf_time'))

98
def Iperf_analyzeV3TCPJson(filename, iperf_tcp_rate_target):
99 100 101 102 103 104 105 106 107 108 109 110
	try:
		with open(filename) as f:
			 results = json.load(f)
		sender_bitrate   = round(results['end']['streams'][0]['sender']['bits_per_second'] / 1000000, 2)
		receiver_bitrate = round(results['end']['streams'][0]['receiver']['bits_per_second'] / 1000000, 2)
		snd_msg = f'Sender Bitrate   : {sender_bitrate} Mbps'
		rcv_msg = f'Receiver Bitrate : {receiver_bitrate} Mbps'
		success = True
		if (iperf_tcp_rate_target is not None):
			if (int(receiver_bitrate) < int(iperf_tcp_rate_target)):
				rcv_msg += f" (too low! < {iperf_tcp_rate_target} Mbps)"
				success = False
111 112
		else:
			rcv_msg += f" (target : {iperf_tcp_rate_target} Mbps)"
113 114 115 116 117 118
		return(success, f'{snd_msg}\n{rcv_msg}')
	except KeyError as e:
		e_msg = results.get('error', f'error report not found in {filename}')
		return (False, f'While parsing Iperf3 results: missing key {e}, {e_msg}')
	except Exception as e:
		return (False, f'While parsing Iperf3 results: generic exception: {e}')
119 120 121 122 123 124 125 126 127 128

def Iperf_analyzeV3BIDIRJson(filename):
	if (not os.path.isfile(filename)):
		return (False, 'Iperf3 Bidir TCP: Log file not present')
	if (os.path.getsize(filename)==0):
		return (False, 'Iperf3 Bidir TCP: Log file is empty')

	with open(filename) as file:
		filename = json.load(file)
		try:
129 130 131 132
			sender_bitrate_ul   = round(filename['end']['streams'][0]['sender']['bits_per_second']/1000000,2)
			receiver_bitrate_ul = round(filename['end']['streams'][0]['receiver']['bits_per_second']/1000000,2)
			sender_bitrate_dl   = round(filename['end']['streams'][1]['sender']['bits_per_second']/1000000,2)
			receiver_bitrate_dl = round(filename['end']['streams'][1]['receiver']['bits_per_second']/1000000,2)
133 134 135 136 137 138 139 140 141
		except Exception as e:
			return (False, 'Could not compute BIDIR bitrate!')

	msg = f'Sender Bitrate DL   : {sender_bitrate_dl} Mbps\n'
	msg += f'Receiver Bitrate DL : {receiver_bitrate_dl} Mbps\n'
	msg += f'Sender Bitrate UL   : {sender_bitrate_ul} Mbps\n'
	msg += f'Receiver Bitrate UL : {receiver_bitrate_ul} Mbps\n'
	return (True, msg)

142
def Iperf_analyzeV3UDP(filename, iperf_bitrate_threshold, iperf_packetloss_threshold, target_bitrate):
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
	if (not os.path.isfile(filename)):
		return (False, 'Iperf3 UDP: Log file not present')
	if (os.path.getsize(filename)==0):
		return (False, 'Iperf3 UDP: Log file is empty')
	sender_bitrate = None
	receiver_bitrate = None
	with open(filename, 'r') as server_file:
		for line in server_file.readlines():
			res_sender = re.search(r'(?P<bitrate>[0-9\.]+)\s+(?P<unit>[KMG]bits\/sec)\s+(?P<jitter>[0-9\.]+\s+ms)\s+(?P<lostPack>\d+)/(?P<sentPack>\d+) \((?P<lost>[0-9\.]+).*?\s+(sender)', line)
			res_receiver = re.search(r'(?P<bitrate>[0-9\.]+)\s+(?P<unit>[KMG]bits\/sec)\s+(?P<jitter>[0-9\.]+\s+ms)\s+(?P<lostPack>\d+)/(?P<receivedPack>\d+) \((?P<lost>[0-9\.]+).*?\s+(receiver)', line)
			if res_sender is not None:
				sender_bitrate = res_sender.group('bitrate')
				sender_unit = res_sender.group('unit')
				sender_jitter = res_sender.group('jitter')
				sender_lostPack = res_sender.group('lostPack')
				sender_sentPack = res_sender.group('sentPack')
				sender_packetloss = res_sender.group('lost')
			if res_receiver is not None:
				receiver_bitrate = res_receiver.group('bitrate')
				receiver_unit = res_receiver.group('unit')
				receiver_jitter = res_receiver.group('jitter')
				receiver_lostPack = res_receiver.group('lostPack')
				receiver_receivedPack = res_receiver.group('receivedPack')
				receiver_packetloss = res_receiver.group('lost')

	if receiver_bitrate is not None and sender_bitrate is not None:
		if sender_unit == 'Kbits/sec':
			sender_bitrate = float(sender_bitrate) / 1000
		if receiver_unit == 'Kbits/sec':
			receiver_bitrate = float(receiver_bitrate) / 1000
173
		br_perf = 100 * float(receiver_bitrate) / float(target_bitrate)
174 175 176
		br_perf = '%.2f ' % br_perf
		sender_bitrate = '%.2f ' % float(sender_bitrate)
		receiver_bitrate = '%.2f ' % float(receiver_bitrate)
177 178
		req_msg = f'Sender Bitrate  : {sender_bitrate} Mbps'
		bir_msg = f'Receiver Bitrate: {receiver_bitrate} Mbps'
179
		brl_msg = f'{br_perf}%'
180 181
		jit_msg = f'Jitter          : {receiver_jitter}'
		pal_msg = f'Packet Loss     : {receiver_packetloss} %'
182 183 184 185 186 187 188 189
		if float(br_perf) < float(iperf_bitrate_threshold):
			brl_msg = f'too low! < {iperf_bitrate_threshold}%'
		if float(receiver_packetloss) > float(iperf_packetloss_threshold):
			pal_msg += f' (too high! > {iperf_packetloss_threshold}%)'
		result = float(br_perf) >= float(iperf_bitrate_threshold) and float(receiver_packetloss) <= float(iperf_packetloss_threshold)
		return (result, f'{req_msg}\n{bir_msg} ({brl_msg})\n{jit_msg}\n{pal_msg}')
	else:
		return (False, 'Could not analyze iperf report')
190

191 192
def Iperf_analyzeV2UDP(server_filename, iperf_bitrate_threshold, iperf_packetloss_threshold, target_bitrate):
		result = None
193
		if (not os.path.isfile(server_filename)):
194 195 196
			return (False, 'Iperf UDP: Server report not found!')
		if (os.path.getsize(server_filename)==0):
			return (False, 'Iperf UDP: Log file is empty')
197
		# Computing the requested bandwidth in float
198
		statusTemplate = r'(?:|\[ *\d+\].*) +0\.0-\s*(?P<duration>[0-9\.]+) +sec +[0-9\.]+ [kKMG]Bytes +(?P<bitrate>[0-9\.]+) (?P<magnitude>[kKMG])bits\/sec +(?P<jitter>[0-9\.]+) ms +(\d+\/ *\d+) +(\((?P<packetloss>[0-9\.]+)%\))'
199 200
		with open(server_filename, 'r') as server_file:
			for line in server_file.readlines():
201
				result = re.search(statusTemplate, str(line))
202 203 204 205 206 207 208 209 210 211
		if result is None:
			return (False, 'Could not parse server report!')
		bitrate = float(result.group('bitrate'))
		magn = result.group('magnitude')
		if magn == "k" or magn == "K":
			bitrate /= 1000
		elif magn == "G": # we assume bitrate in Mbps, therefore it must be G now
			bitrate *= 1000
		jitter = float(result.group('jitter'))
		packetloss = float(result.group('packetloss'))
212
		br_perf = float(bitrate)/float(target_bitrate) * 100
213 214 215
		br_perf = '%.2f ' % br_perf

		result = float(br_perf) >= float(iperf_bitrate_threshold) and float(packetloss) <= float(iperf_packetloss_threshold)
216 217
		req_msg = f'Req Bitrate : {target_bitrate}'
		bir_msg = f'Bitrate     : {bitrate}'
218 219 220
		brl_msg = f'Bitrate Perf: {br_perf} %'
		if float(br_perf) < float(iperf_bitrate_threshold):
			brl_msg += f' (too low! <{iperf_bitrate_threshold}%)'
221
		jit_msg = f'Jitter      : {jitter}'
222 223 224 225 226
		pal_msg = f'Packet Loss : {packetloss}'
		if float(packetloss) > float(iperf_packetloss_threshold):
			pal_msg += f' (too high! >{self.iperf_packetloss_threshold}%)'
		return (result, f'{req_msg}\n{bir_msg}\n{brl_msg}\n{jit_msg}\n{pal_msg}')

227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
#-----------------------------------------------------------
# OaiCiTest Class Definition
#-----------------------------------------------------------
class OaiCiTest():
	
	def __init__(self):
		self.ranRepository = ''
		self.ranBranch = ''
		self.ranCommitID = ''
		self.ranAllowMerge = False
		self.ranTargetBranch = ''

		self.FailReportCnt = 0
		self.testCase_id = ''
		self.testXMLfiles = []
242 243 244
		self.testUnstable = False
		self.testMinStableId = '999999'
		self.testStabilityPointReached = False
245 246 247
		self.desc = ''
		self.ping_args = ''
		self.ping_packetloss_threshold = ''
248
		self.ping_rttavg_threshold =''
249 250
		self.iperf_args = ''
		self.iperf_packetloss_threshold = ''
hardy's avatar
hardy committed
251
		self.iperf_bitrate_threshold = ''
252 253
		self.iperf_profile = ''
		self.iperf_options = ''
254
		self.iperf_tcp_rate_target = ''
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
		self.nbMaxUEtoAttach = -1
		self.UEDevices = []
		self.UEDevicesStatus = []
		self.UEDevicesRemoteServer = []
		self.UEDevicesRemoteUser = []
		self.UEDevicesOffCmd = []
		self.UEDevicesOnCmd = []
		self.UEDevicesRebootCmd = []
		self.idle_sleep_time = 0
		self.x2_ho_options = 'network'
		self.x2NbENBs = 0
		self.x2ENBBsIds = []
		self.x2ENBConnectedUEs = []
		self.repeatCounts = []
		self.finalStatus = False
		self.UEIPAddress = ''
		self.UEUserName = ''
		self.UEPassword = ''
		self.UE_instance = 0
		self.UESourceCodePath = ''
		self.UELogFile = ''
		self.Build_OAI_UE_args = ''
		self.Initialize_OAI_UE_args = ''
		self.clean_repository = True
		self.air_interface=''
280
		self.ue_ids = []
281
		self.svr_id = None
282
		self.cmd_prefix = '' # prefix before {lte,nr}-uesoftmodem
283

hardy's avatar
hardy committed
284

285 286 287 288
	def BuildOAIUE(self,HTML):
		if self.UEIPAddress == '' or self.ranRepository == '' or self.ranBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
hardy's avatar
hardy committed
289
		SSH = sshconnection.SSHConnection()
290 291 292 293 294 295 296 297 298 299
		SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
		result = re.search('--nrUE', self.Build_OAI_UE_args)
		if result is not None:
			self.air_interface='nr-uesoftmodem'
			ue_prefix = 'NR '
		else:
			self.air_interface='lte-uesoftmodem'
			ue_prefix = ''
		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
		if result is not None:
300
			full_ran_repo_name = self.ranRepository.replace('git/', 'git')
301 302
		else:
			full_ran_repo_name = self.ranRepository + '.git'
303 304 305
		SSH.command(f'mkdir -p {self.UESourceCodePath}', '\$', 5)
		SSH.command(f'cd {self.UESourceCodePath}', '\$', 5)
		SSH.command(f'if [ ! -e .git ]; then stdbuf -o0 git clone {full_ran_repo_name} .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
306 307 308 309 310 311 312 313
		# here add a check if git clone or git fetch went smoothly
		SSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		SSH.command('git config user.name "OAI Jenkins"', '\$', 5)
		if self.clean_repository:
			SSH.command('ls *.txt', '\$', 5)
			result = re.search('LAST_BUILD_INFO', SSH.getBefore())
			if result is not None:
				mismatch = False
314
				SSH.command('grep --colour=never SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
315 316 317
				result = re.search(self.ranCommitID, SSH.getBefore())
				if result is None:
					mismatch = True
318
				SSH.command('grep --colour=never MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
319 320 321 322
				if self.ranAllowMerge:
					result = re.search('YES', SSH.getBefore())
					if result is None:
						mismatch = True
323
					SSH.command('grep --colour=never TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338
					if self.ranTargetBranch == '':
						result = re.search('develop', SSH.getBefore())
					else:
						result = re.search(self.ranTargetBranch, SSH.getBefore())
					if result is None:
						mismatch = True
				else:
					result = re.search('NO', SSH.getBefore())
					if result is None:
						mismatch = True
				if not mismatch:
					SSH.close()
					HTML.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', CONST.ALL_PROCESSES_OK)
					return

339
			SSH.command(f'echo {self.UEPassword} | sudo -S git clean -x -d -ff', '\$', 30)
340 341 342

		# if the commit ID is provided use it to point to it
		if self.ranCommitID != '':
343
			SSH.command(f'git checkout -f {self.ranCommitID}', '\$', 30)
344 345 346 347 348
		# if the branch is not develop, then it is a merge request and we need to do 
		# the potential merge. Note that merge conflicts should already been checked earlier
		if self.ranAllowMerge:
			if self.ranTargetBranch == '':
				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
349
					SSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30)
350
			else:
351 352
				logging.debug(f'Merging with the target branch: {self.ranTargetBranch}')
				SSH.command(f'git merge --ff origin/{self.ranTargetBranch} -m "Temporary merge for CI"', '\$', 30)
353 354 355 356 357
		SSH.command('source oaienv', '\$', 5)
		SSH.command('cd cmake_targets', '\$', 5)
		SSH.command('mkdir -p log', '\$', 5)
		SSH.command('chmod 777 log', '\$', 5)
		# no need to remove in log (git clean did the trick)
358
		SSH.command(f'stdbuf -o0 ./build_oai {self.Build_OAI_UE_args} 2>&1 | stdbuf -o0 tee compile_oai_ue.log', 'Bypassing the Tests|build have failed', 1200)
359 360 361 362 363 364
		SSH.command('ls ran_build/build', '\$', 3)
		SSH.command('ls ran_build/build', '\$', 3)
		buildStatus = True
		result = re.search(self.air_interface, SSH.getBefore())
		if result is None:
			buildStatus = False
365 366 367
		SSH.command(f'mkdir -p build_log_{self.testCase_id}', '\$', 5)
		SSH.command(f'mv log/* build_log_{self.testCase_id}', '\$', 5)
		SSH.command(f'mv compile_oai_ue.log build_log_{self.testCase_id}', '\$', 5)
368 369
		if buildStatus:
			# Generating a BUILD INFO file
370 371
			SSH.command(f'echo "SRC_BRANCH: {self.ranBranch}" > ../LAST_BUILD_INFO.txt', '\$', 2)
			SSH.command(f'echo "SRC_COMMIT: {self.ranCommitID}" >> ../LAST_BUILD_INFO.txt', '\$', 2)
372 373 374 375 376
			if self.ranAllowMerge:
				SSH.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				if self.ranTargetBranch == '':
					SSH.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
				else:
377
					SSH.command(f'echo "TGT_BRANCH: {self.ranTargetBranch}" >> ../LAST_BUILD_INFO.txt', '\$', 2)
378 379 380 381 382 383 384 385 386
			else:
				SSH.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
			SSH.close()
			HTML.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', CONST.ALL_PROCESSES_OK, 'OAI UE')
		else:
			SSH.close()
			logging.error('\u001B[1m Building OAI UE Failed\u001B[0m')
			HTML.CreateHtmlTestRow(self.Build_OAI_UE_args, 'KO', CONST.ALL_PROCESSES_OK, 'OAI UE')
			HTML.CreateHtmlTabFooter(False)
387
			self.ConditionalExit()
388

389

390
	def InitializeUE(self, HTML):
391
		ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
392
		messages = []
393 394
		with concurrent.futures.ThreadPoolExecutor() as executor:
			futures = [executor.submit(ue.initialize) for ue in ues]
395 396 397
			for f, ue in zip(futures, ues):
				uename = f'UE {ue.getName()}'
				messages.append(f'{uename}: initialized' if f.result() else f'{uename}: ERROR during Initialization')
398 399 400
			[f.result() for f in futures]
		HTML.CreateHtmlTestRowQueue('N/A', 'OK', messages)

Robert Schmidt's avatar
Robert Schmidt committed
401
	def InitializeOAIUE(self,HTML,RAN,EPC,CONTAINERS):
402 403 404
		if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
405 406

			
407 408 409 410 411 412 413 414
		if self.air_interface == 'lte-uesoftmodem':
			result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
			if result is None:
				check_eNB = True
				check_OAI_UE = False
			UE_prefix = ''
		else:
			UE_prefix = 'NR '
hardy's avatar
hardy committed
415
		SSH = sshconnection.SSHConnection()
416
		SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
417
		SSH.command(f'cd {self.UESourceCodePath}', '\$', 5)
418 419 420 421 422 423 424 425 426 427 428 429 430
		# Initialize_OAI_UE_args usually start with -C and followed by the location in repository
		SSH.command('source oaienv', '\$', 5)
		SSH.command('cd cmake_targets/ran_build/build', '\$', 5)
		if self.air_interface == 'lte-uesoftmodem':
			result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
			# We may have to regenerate the .u* files
			if result is None:
				SSH.command('ls /tmp/*.sed', '\$', 5)
				result = re.search('adapt_usim_parameters', SSH.getBefore())
				if result is not None:
					SSH.command('sed -f /tmp/adapt_usim_parameters.sed ../../../openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf > ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf', '\$', 5)
				else:
					SSH.command('sed -e "s#93#92#" -e "s#8baf473f2f8fd09487cccbd7097c6862#fec86ba6eb707ed08905757b1bb44b8f#" -e "s#e734f8734007d6c5ce7a0508809e7e9c#C42449363BBAD02B66D16BC975D77CC1#" ../../../openair3/NAS/TOOLS/ue_eurecom_test_sfr.conf > ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf', '\$', 5)
431 432
				SSH.command(f'echo {self.UEPassword} | sudo -S rm -Rf .u*', '\$', 5)
				SSH.command(f'echo {self.UEPassword} | sudo -S ../../nas_sim_tools/build/conf2uedata -c ../../../openair3/NAS/TOOLS/ci-ue_eurecom_test_sfr.conf -o .', '\$', 5)
433
		else:
434 435
			SSH.command(f'if [ -e rbconfig.raw ]; then echo {self.UEPassword} | sudo -S rm rbconfig.raw; fi', '\$', 5)
			SSH.command(f'if [ -e reconfig.raw ]; then echo {self.UEPassword} | sudo -S rm reconfig.raw; fi', '\$', 5)
436 437 438 439 440 441 442
			# Copy the RAW files from gNB running directory (maybe on another machine)
			copyin_res = SSH.copyin(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, RAN.eNBSourceCodePath + '/cmake_targets/rbconfig.raw', '.')
			if (copyin_res == 0):
				SSH.copyout(self.UEIPAddress, self.UEUserName, self.UEPassword, './rbconfig.raw', self.UESourceCodePath + '/cmake_targets/ran_build/build')
			copyin_res = SSH.copyin(RAN.eNBIPAddress, RAN.eNBUserName, RAN.eNBPassword, RAN.eNBSourceCodePath + '/cmake_targets/reconfig.raw', '.')
			if (copyin_res == 0):
				SSH.copyout(self.UEIPAddress, self.UEUserName, self.UEPassword, './reconfig.raw', self.UESourceCodePath + '/cmake_targets/ran_build/build')
443
		SSH.command(f'echo "ulimit -c unlimited && {self.cmd_prefix} ./{self.air_interface} {self.Initialize_OAI_UE_args}" > ./my-lte-uesoftmodem-run{self.UE_instance}.sh', '\$', 5)
444 445 446
		SSH.command(f'chmod 775 ./my-lte-uesoftmodem-run {self.UE_instance}.sh', '\$', 5)
		SSH.command(f'echo {self.UEPassword} | sudo -S rm -Rf {self.UESourceCodePath}/cmake_targets/ue_{self.testCase_id}.log', '\$', 5)
		self.UELogFile = f'ue_{self.testCase_id}.log'
447 448 449 450 451 452 453

		# We are now looping several times to hope we really sync w/ an eNB
		doOutterLoop = True
		outterLoopCounter = 5
		gotSyncStatus = True
		fullSyncStatus = True
		while (doOutterLoop):
454 455 456
			SSH.command(f'cd {self.UESourceCodePath}/cmake_targets/ran_build/build', '\$', 5)
			SSH.command(f'echo {self.UEPassword} | sudo -S rm -Rf {self.UESourceCodePath}/cmake_targets/ue_{self.testCase_id}.log', '\$', 5)
			SSH.command(f'echo $USER; nohup sudo -E stdbuf -o0 ./my-lte-uesoftmodem-run {self.UE_instance}.sh > {self.UESourceCodePath}/cmake_targets/ue_{self.testCase_id}.log 2>&1 &', self.UEUserName, 5)
457 458 459 460 461 462 463 464 465 466 467 468 469 470
			time.sleep(6)
			SSH.command('cd ../..', '\$', 5)
			doLoop = True
			loopCounter = 10
			gotSyncStatus = True
			# the 'got sync' message is for the UE threads synchronization
			while (doLoop):
				loopCounter = loopCounter - 1
				if (loopCounter == 0):
					# Here should never occur
					logging.error('"got sync" message never showed!')
					gotSyncStatus = False
					doLoop = False
					continue
471
				SSH.command(f'stdbuf -o0 cat ue_{self.testCase_id}.log | egrep --text --color=never -i "wait|sync"', '\$', 4)
472 473 474 475 476 477 478 479 480 481 482 483 484 485
				if self.air_interface == 'nr-uesoftmodem':
					result = re.search('Starting sync detection', SSH.getBefore())
				else:
					result = re.search('got sync', SSH.getBefore())
				if result is None:
					time.sleep(10)
				else:
					doLoop = False
					logging.debug('Found "got sync" message!')
			if gotSyncStatus == False:
				# we certainly need to stop the lte-uesoftmodem process if it is still running!
				SSH.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
				result = re.search('-uesoftmodem', SSH.getBefore())
				if result is not None:
486
					SSH.command(f'echo {self.UEPassword} | sudo -S killall --signal=SIGINT -r *-uesoftmodem', '\$', 4)
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
					time.sleep(3)
				outterLoopCounter = outterLoopCounter - 1
				if (outterLoopCounter == 0):
					doOutterLoop = False
				continue
			# We are now checking if sync w/ eNB DOES NOT OCCUR
			# Usually during the cell synchronization stage, the UE returns with No cell synchronization message
			# That is the case for LTE
			# In NR case, it's a positive message that will show if synchronization occurs
			doLoop = True
			if self.air_interface == 'nr-uesoftmodem':
				loopCounter = 10
			else:
				# We are now checking if sync w/ eNB DOES NOT OCCUR
				# Usually during the cell synchronization stage, the UE returns with No cell synchronization message
				loopCounter = 10
			while (doLoop):
				loopCounter = loopCounter - 1
				if (loopCounter == 0):
					if self.air_interface == 'nr-uesoftmodem':
						# Here we do have great chances that UE did NOT cell-sync w/ gNB
						doLoop = False
						fullSyncStatus = False
						logging.debug('Never seen the NR-Sync message (Measured Carrier Frequency) --> try again')
						time.sleep(6)
						# Stopping the NR-UE  
						SSH.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
						result = re.search('nr-uesoftmodem', SSH.getBefore())
						if result is not None:
516
							SSH.command(f'echo {self.UEPassword} | sudo -S killall --signal=SIGINT nr-uesoftmodem', '\$', 4)
517 518 519 520 521 522 523
						time.sleep(6)
					else:
						# Here we do have a great chance that the UE did cell-sync w/ eNB
						doLoop = False
						doOutterLoop = False
						fullSyncStatus = True
						continue
524
				SSH.command(f'stdbuf -o0 cat ue_{self.testCase_id}.log | egrep --text --color=never -i "wait|sync|Frequency"', '\$', 4)
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546
				if self.air_interface == 'nr-uesoftmodem':
					# Positive messaging -->
					result = re.search('Measured Carrier Frequency', SSH.getBefore())
					if result is not None:
						doLoop = False
						doOutterLoop = False
						fullSyncStatus = True
					else:
						time.sleep(6)
				else:
					# Negative messaging -->
					result = re.search('No cell synchronization found', SSH.getBefore())
					if result is None:
						time.sleep(6)
					else:
						doLoop = False
						fullSyncStatus = False
						logging.debug('Found: "No cell synchronization" message! --> try again')
						time.sleep(6)
						SSH.command('ps -aux | grep --text --color=never softmodem | grep -v grep', '\$', 4)
						result = re.search('lte-uesoftmodem', SSH.getBefore())
						if result is not None:
547
							SSH.command(f'echo {self.UEPassword} | sudo -S killall --signal=SIGINT lte-uesoftmodem', '\$', 4)
548 549 550 551 552 553 554 555 556 557 558 559
			outterLoopCounter = outterLoopCounter - 1
			if (outterLoopCounter == 0):
				doOutterLoop = False

		if fullSyncStatus and gotSyncStatus:
			doInterfaceCheck = False
			if self.air_interface == 'lte-uesoftmodem':
				result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
				if result is None:
					doInterfaceCheck = True
			# For the moment, only in explicit noS1 without kernel module (ie w/ tunnel interface)
			if self.air_interface == 'nr-uesoftmodem':
560
				result = re.search('--noS1', str(self.Initialize_OAI_UE_args))
561 562 563 564 565 566
				if result is not None:
					doInterfaceCheck = True
			if doInterfaceCheck:
				SSH.command('ifconfig oaitun_ue1', '\$', 4)
				SSH.command('ifconfig oaitun_ue1', '\$', 4)
				# ifconfig output is different between ubuntu 16 and ubuntu 18
567
				result = re.search('inet addr:[0-9]|inet [0-9]', SSH.getBefore())
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
				if result is not None:
					logging.debug('\u001B[1m oaitun_ue1 interface is mounted and configured\u001B[0m')
					tunnelInterfaceStatus = True
				else:
					logging.debug(SSH.getBefore())
					logging.error('\u001B[1m oaitun_ue1 interface is either NOT mounted or NOT configured\u001B[0m')
					tunnelInterfaceStatus = False
				if RAN.eNBmbmsEnables[0]:
					SSH.command('ifconfig oaitun_uem1', '\$', 4)
					result = re.search('inet addr', SSH.getBefore())
					if result is not None:
						logging.debug('\u001B[1m oaitun_uem1 interface is mounted and configured\u001B[0m')
						tunnelInterfaceStatus = tunnelInterfaceStatus and True
					else:
						logging.error('\u001B[1m oaitun_uem1 interface is either NOT mounted or NOT configured\u001B[0m')
						tunnelInterfaceStatus = False
			else:
				tunnelInterfaceStatus = True
		else:
			tunnelInterfaceStatus = True

		SSH.close()
		if fullSyncStatus and gotSyncStatus and tunnelInterfaceStatus:
			HTML.CreateHtmlTestRow(self.air_interface + ' ' + self.Initialize_OAI_UE_args, 'OK', CONST.ALL_PROCESSES_OK, 'OAI UE')
			logging.debug('\u001B[1m Initialize OAI UE Completed\u001B[0m')
		else:
			if self.air_interface == 'lte-uesoftmodem':
				if RAN.eNBmbmsEnables[0]:
					HTML.htmlUEFailureMsg='oaitun_ue1/oaitun_uem1 interfaces are either NOT mounted or NOT configured'
				else:
					HTML.htmlUEFailureMsg='oaitun_ue1 interface is either NOT mounted or NOT configured'
				HTML.CreateHtmlTestRow(self.air_interface + ' ' + self.Initialize_OAI_UE_args, 'KO', CONST.OAI_UE_PROCESS_NO_TUNNEL_INTERFACE, 'OAI UE')
			else:
				HTML.htmlUEFailureMsg='nr-uesoftmodem did NOT synced'
				HTML.CreateHtmlTestRow(self.air_interface + ' ' +  self.Initialize_OAI_UE_args, 'KO', CONST.OAI_UE_PROCESS_COULD_NOT_SYNC, 'OAI UE')
			logging.error('\033[91mInitialize OAI UE Failed! \033[0m')
Robert Schmidt's avatar
Robert Schmidt committed
604
			self.AutoTerminateUEandeNB(HTML,RAN,EPC,CONTAINERS)
605

606
	def AttachUE(self, HTML, RAN, EPC, CONTAINERS):
607
		ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
608 609 610 611 612 613 614 615
		with concurrent.futures.ThreadPoolExecutor() as executor:
			futures = [executor.submit(ue.attach) for ue in ues]
			attached = [f.result() for f in futures]
			futures = [executor.submit(ue.checkMTU) for ue in ues]
			mtus = [f.result() for f in futures]
			messages = [f"UE {ue.getName()}: {ue.getIP()}" for ue in ues]
		if all(attached) and all(mtus):
			HTML.CreateHtmlTestRowQueue('N/A', 'OK', messages)
616
		else:
617 618 619 620 621
			logging.error(f'error attaching or wrong MTU: attached {attached}, mtus {mtus}')
			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ["Could not retrieve UE IP address(es) or MTU(s) wrong!"])
			self.AutoTerminateUEandeNB(HTML, RAN, EPC, CONTAINERS)

	def DetachUE(self, HTML):
622
		ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
623 624 625 626 627
		with concurrent.futures.ThreadPoolExecutor() as executor:
			futures = [executor.submit(ue.detach) for ue in ues]
			[f.result() for f in futures]
			messages = [f"UE {ue.getName()}: detached" for ue in ues]
		HTML.CreateHtmlTestRowQueue('NA', 'OK', messages)
628

629
	def DataDisableUE(self, HTML):
630
		ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
631 632 633 634 635 636
		with concurrent.futures.ThreadPoolExecutor() as executor:
			futures = [executor.submit(ue.dataDisable) for ue in ues]
			status = [f.result() for f in futures]
		if all(status):
			messages = [f"UE {ue.getName()}: data disabled" for ue in ues]
			HTML.CreateHtmlTestRowQueue('NA', 'OK', messages)
637
		else:
638 639 640 641
			logging.error(f'error enabling data: {status}')
			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ["Could not disable UE data!"])

	def DataEnableUE(self, HTML):
642
		ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
643 644 645 646 647 648 649 650 651 652 653 654
		logging.debug(f'disabling data for UEs {ues}')
		with concurrent.futures.ThreadPoolExecutor() as executor:
			futures = [executor.submit(ue.dataEnable) for ue in ues]
			status = [f.result() for f in futures]
		if all(status):
			messages = [f"UE {ue.getName()}: data enabled" for ue in ues]
			HTML.CreateHtmlTestRowQueue('NA', 'OK', messages)
		else:
			logging.error(f'error enabling data: {status}')
			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ["Could not enable UE data!"])

	def CheckStatusUE(self,HTML):
655
		ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
656 657 658 659 660 661
		logging.debug(f'checking status of UEs {ues}')
		messages = []
		with concurrent.futures.ThreadPoolExecutor() as executor:
			futures = [executor.submit(ue.check) for ue in ues]
			messages = [f.result() for f in futures]
		HTML.CreateHtmlTestRowQueue('NA', 'OK', messages)
662

663
	def Ping_common(self, EPC, ue, RAN, printLock):
664 665
		# Launch ping on the EPC side (true for ltebox and old open-air-cn)
		ping_status = 0
666 667 668
		ueIP = ue.getIP()
		if not ueIP:
			return (False, f"UE {ue.getName()} has no IP address")
669 670
		ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log'
		ping_time = re.findall("-c *(\d+)",str(self.ping_args))
671 672 673 674 675 676 677 678 679 680 681 682
		local_ping_log_file = f'{os.getcwd()}/{ping_log_file}'
		# if has pattern %cn_ip%, replace with core IP address, else we assume the IP is present
		if re.search('%cn_ip%', self.ping_args):
			#target address is different depending on EPC type
			if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE):
				self.ping_args = re.sub('%cn_ip%', EPC.MmeIPAddress, self.ping_args)
			elif re.match('OAICN5G', EPC.Type, re.IGNORECASE):
				self.ping_args = re.sub('%cn_ip%', EPC.MmeIPAddress, self.ping_args)
			elif re.match('OC-OAI-CN5G', EPC.Type, re.IGNORECASE):
				self.ping_args = re.sub('%cn_ip%', '172.21.6.100', self.ping_args)
			else:
				self.ping_args = re.sub('%cn_ip%', EPC.IPAddress, self.ping_args)
683 684
		#ping from module NIC rather than IP address to make sure round trip is over the air
		interface = f'-I {ue.getIFName()}' if ue.getIFName() else ''
685
		ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} 2>&1 | tee /tmp/{ping_log_file}'
686 687
		cmd = cls_cmd.getConnection(ue.getHost())
		response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5)
688
		ue_header = f'UE {ue.getName()} ({ueIP})'
689
		if response.returncode != 0:
690 691 692
			message = ue_header + ': ping crashed: TIMEOUT?'
			logging.error('\u001B[1;37;41m ' + message + ' \u001B[0m')
			return (False, message)
693

694 695
		#copy the ping log file to have it locally for analysis (ping stats)
		cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=local_ping_log_file)
696
		cmd.close()
697

698
		with open(local_ping_log_file, 'r') as f:
699 700 701
			ping_output = "".join(f.readlines())
		result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', ping_output)
		if result is None:
702
			message = ue_header + ': Packet Loss Not Found!'
703
			logging.error(f'\u001B[1;37;41m {message} \u001B[0m')
704
			return (False, message)
705 706 707
		packetloss = result.group('packetloss')
		result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', ping_output)
		if result is None:
708
			message = ue_header + ': Ping RTT_Min RTT_Avg RTT_Max Not Found!'
709
			logging.error(f'\u001B[1;37;41m {message} \u001B[0m')
710
			return (False, message)
711 712 713 714
		rtt_min = result.group('rtt_min')
		rtt_avg = result.group('rtt_avg')
		rtt_max = result.group('rtt_max')

715 716 717 718
		pal_msg = f'Packet Loss: {packetloss}%'
		min_msg = f'RTT(Min)   : {rtt_min} ms'
		avg_msg = f'RTT(Avg)   : {rtt_avg} ms'
		max_msg = f'RTT(Max)   : {rtt_max} ms'
719

720 721
		# adding a lock for cleaner display in command line
		printLock.acquire()
722
		logging.info(f'\u001B[1;37;44m ping result for {ue_header} \u001B[0m')
723 724 725 726
		logging.info(f'\u001B[1;34m    {pal_msg} \u001B[0m')
		logging.info(f'\u001B[1;34m    {min_msg} \u001B[0m')
		logging.info(f'\u001B[1;34m    {avg_msg} \u001B[0m')
		logging.info(f'\u001B[1;34m    {max_msg} \u001B[0m')
727

728
		message = f'{ue_header}\n{pal_msg}\n{min_msg}\n{avg_msg}\n{max_msg}'
729 730

		#checking packet loss compliance
731 732 733
		if float(packetloss) > float(self.ping_packetloss_threshold):
			message += '\nPacket Loss too high'
			logging.error(f'\u001B[1;37;41m Packet Loss too high; Target: {self.ping_packetloss_threshold}%\u001B[0m')
734
			printLock.release()
735 736 737 738 739
			return (False, message)
		elif float(packetloss) > 0:
			message += '\nPacket Loss is not 0%'
			logging.info('\u001B[1;30;43m Packet Loss is not 0% \u001B[0m')

740
		if self.ping_rttavg_threshold != '':
741 742
			if float(rtt_avg) > float(self.ping_rttavg_threshold):
				ping_rttavg_error_msg = f'RTT(Avg) too high: {rtt_avg} ms; Target: {self.ping_rttavg_threshold} ms'
743
				message += f'\n {ping_rttavg_error_msg}'
744
				logging.error('\u001B[1;37;41m'+ ping_rttavg_error_msg +' \u001B[0m')
745
				printLock.release()
746
				return (False, message)
747
		printLock.release()
748 749

		return (True, message)
750

Robert Schmidt's avatar
Robert Schmidt committed
751
	def Ping(self,HTML,RAN,EPC,CONTAINERS):
752 753 754
		if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
755

756 757
		if self.ue_ids == []:
			raise Exception("no module names in self.ue_ids provided")
hardy's avatar
hardy committed
758

759
		ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
760
		logging.debug(ues)
761
		pingLock = Lock()
762
		with concurrent.futures.ThreadPoolExecutor() as executor:
763
			futures = [executor.submit(self.Ping_common, EPC, ue, RAN, pingLock) for ue in ues]
764 765 766 767 768
			results = [f.result() for f in futures]
			# each result in results is a tuple, first member goes to successes, second to messages
			successes, messages = map(list, zip(*results))
		if len(successes) == len(ues) and all(successes):
			HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', messages)
769
		else:
770 771
			HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', messages)
			self.AutoTerminateUEandeNB(HTML,RAN,EPC,CONTAINERS)
772

Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
773
	def Iperf_Module(self, EPC, ue, svr, RAN, idx, ue_num, CONTAINERS):
774 775 776
		ueIP = ue.getIP()
		if not ueIP:
			return (False, f"UE {ue.getName()} has no IP address")
Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
777 778 779
		svrIP = svr.getIP()
		if not svrIP:
			return (False, f"Iperf server {ue.getName()} has no IP address")
Robert Schmidt's avatar
Robert Schmidt committed
780

Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
781
		runIperf3Server = svr.getRunIperf3Server()
Robert Schmidt's avatar
Robert Schmidt committed
782
		iperf_opt = self.iperf_args
Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
783 784
		jsonReport = "--json"
		serverReport = ""
Robert Schmidt's avatar
Robert Schmidt committed
785
		udpIperf = re.search('-u', iperf_opt) is not None
Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
786 787
		bidirIperf = re.search('--bidir', iperf_opt) is not None
		client_filename = f'iperf_client_{self.testCase_id}_{ue.getName()}.log'
788 789
		ymlPath = CONTAINERS.yamlPath[0].split('/')
		logPath = f'../cmake_targets/log/{ymlPath[1]}'
Robert Schmidt's avatar
Robert Schmidt committed
790
		if udpIperf:
791
			target_bitrate, iperf_opt = Iperf_ComputeModifiedBW(idx, ue_num, self.iperf_profile, self.iperf_args)
Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
792 793 794 795
			# note: for UDP testing we don't want to use json report - reports 0 Mbps received bitrate
			jsonReport = ""
			# note: enable server report collection on the UE side, no need to store and collect server report separately on the server side
			serverReport = "--get-server-output"
Robert Schmidt's avatar
Robert Schmidt committed
796
			logging.info(f'iperf options modified from "{self.iperf_args}" to "{iperf_opt}" for {ue.getName()}')
Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
797
		iperf_time = Iperf_ComputeTime(self.iperf_args)
Robert Schmidt's avatar
Robert Schmidt committed
798
		# hack: the ADB UEs don't have iperf in $PATH, so we need to hardcode for the moment
Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
799
		iperf_ue = '/data/local/tmp/iperf3' if re.search('adb', ue.getName()) else 'iperf3'
800
		ue_header = f'UE {ue.getName()} ({ueIP})'
801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821
		# create log directory on executor node
		with cls_cmd.getConnection('localhost') as local:
			local.run(f'mkdir -p {logPath}')
		with cls_cmd.getConnection(ue.getHost()) as cmd_ue, cls_cmd.getConnection(EPC.IPAddress) as cmd_svr:
			port = 5002 + idx
			# note: some core setups start an iperf3 server automatically, indicated in ci_infra by runIperf3Server: False`
			t = iperf_time * 2.5
			cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False)
			if runIperf3Server:
				cmd_svr.run(f'{svr.getCmdPrefix()} nohup timeout -vk3 {t} iperf3 -s -B {svrIP} -p {port} -1 {jsonReport} &', timeout=t)
			cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> /tmp/{client_filename}', timeout=t)
			localPath = f'{os.getcwd()}'
			# note: copy iperf3 log to the directory for log collection, used by pipelines running on executor machine
			cmd_ue.copyin(f'/tmp/{client_filename}', f'{localPath}/{logPath}/{client_filename}')
			# note: copy iperf3 log to the current directory for log analysis and log collection
			cmd_ue.copyin(f'/tmp/{client_filename}', f'{localPath}/{client_filename}')
			cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False)
		if udpIperf:
			status, msg = Iperf_analyzeV3UDP(client_filename, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate)
		elif bidirIperf:
			status, msg = Iperf_analyzeV3BIDIRJson(client_filename)
Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
822
		else:
823
			status, msg = Iperf_analyzeV3TCPJson(client_filename, self.iperf_tcp_rate_target)
824

825 826 827 828
		logging.info(f'\u001B[1;37;45m iperf result for {ue_header}\u001B[0m')
		for l in msg.split('\n'):
			logging.info(f'\u001B[1;35m    {l} \u001B[0m')
		return (status, f'{ue_header}\n{msg}')
829

830 831
	def IperfNoS1(self,HTML,RAN,EPC,CONTAINERS):
		raise 'IperfNoS1 not implemented'
832

Robert Schmidt's avatar
Robert Schmidt committed
833
	def Iperf(self,HTML,RAN,EPC,CONTAINERS):
834 835
		result = re.search('noS1', str(RAN.Initialize_eNB_args))
		if result is not None:
Robert Schmidt's avatar
Robert Schmidt committed
836
			self.IperfNoS1(HTML,RAN,EPC,CONTAINERS)
837
			return
Robert Schmidt's avatar
Robert Schmidt committed
838
		if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '':
839 840
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
Robert Schmidt's avatar
Robert Schmidt committed
841

Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
842 843 844 845
		logging.debug(f'Iperf: iperf_args "{self.iperf_args}" iperf_packetloss_threshold "{self.iperf_packetloss_threshold}" iperf_bitrate_threshold "{self.iperf_bitrate_threshold}" iperf_profile "{self.iperf_profile}" iperf_options "{self.iperf_options}"')

		if self.ue_ids == [] or self.svr_id == None:
			raise Exception("no module names in self.ue_ids or/and self.svr_id provided")
846

847 848
		ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
		svr = cls_module.Module_UE(self.svr_id)
Robert Schmidt's avatar
Robert Schmidt committed
849
		logging.debug(ues)
850
		with concurrent.futures.ThreadPoolExecutor() as executor:
Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
851
			futures = [executor.submit(self.Iperf_Module, EPC, ue, svr, RAN, i, len(ues), CONTAINERS) for i, ue in enumerate(ues)]
852 853 854 855 856
			results = [f.result() for f in futures]
			# each result in results is a tuple, first member goes to successes, second to messages
			successes, messages = map(list, zip(*results))
		if len(successes) == len(ues) and all(successes):
			HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', messages)
857
		else:
858 859
			HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', messages)
			self.AutoTerminateUEandeNB(HTML,RAN,EPC,CONTAINERS)
860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899

	def Iperf2_Unidir(self,HTML,RAN,EPC,CONTAINERS):
		if self.ue_ids == [] or self.svr_id == None or len(self.ue_ids) != 1:
			raise Exception("no module names in self.ue_ids or/and self.svr_id provided, multi UE scenario not supported")
		ue = cls_module.Module_UE(self.ue_ids[0].strip())
		svr = cls_module.Module_UE(self.svr_id)
		ueIP = ue.getIP()
		if not ueIP:
			return (False, f"UE {ue.getName()} has no IP address")
		svrIP = svr.getIP()
		if not svrIP:
			return (False, f"Iperf server {ue.getName()} has no IP address")
		server_filename = f'iperf_server_{self.testCase_id}_{ue.getName()}.log'
		ymlPath = CONTAINERS.yamlPath[0].split('/')
		logPath = f'../cmake_targets/log/{ymlPath[-1]}'
		iperf_time = Iperf_ComputeTime(self.iperf_args)
		target_bitrate, iperf_opt = Iperf_ComputeModifiedBW(0, 1, self.iperf_profile, self.iperf_args)
		t = iperf_time*2.5
		with cls_cmd.getConnection('localhost') as local:
			local.run(f'mkdir -p {logPath}')
		with cls_cmd.getConnection(ue.getHost()) as cmd_ue, cls_cmd.getConnection(EPC.IPAddress) as cmd_svr:
			cmd_ue.run(f'rm /tmp/{server_filename}', reportNonZero=False)
			cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} iperf -B {ueIP} -s -u -i1 >> /tmp/{server_filename} &', timeout=t)
			cmd_svr.run(f'{svr.getCmdPrefix()} timeout -vk3 {t} iperf -c {ueIP} -B {svrIP} {iperf_opt} -i1', timeout=t)
			localPath = f'{os.getcwd()}'
			# note: copy iperf2 log to the directory for log collection
			cmd_ue.copyin(f'/tmp/{server_filename}', f'{localPath}/{logPath}/{server_filename}')
			# note: copy iperf2 log to the current directory for log analysis and log collection
			cmd_ue.copyin(f'/tmp/{server_filename}', f'{localPath}/{server_filename}')
			cmd_ue.run(f'rm /tmp/{server_filename}', reportNonZero=False)
		success, msg = Iperf_analyzeV2UDP(server_filename, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate)
		ue_header = f'UE {ue.getName()} ({ueIP})'
		logging.info(f'\u001B[1;37;45m iperf result for {ue_header}\u001B[0m')
		for l in msg.split('\n'):
			logging.info(f'\u001B[1;35m	{l} \u001B[0m')
		if success:
			HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', [f'{ue_header}\n{msg}'])
		else:
			HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', [f'{ue_header}\n{msg}'])
			self.AutoTerminateUEandeNB(HTML,RAN,EPC,CONTAINERS)
900

901
	def AnalyzeLogFile_UE(self, UElogFile,HTML,RAN):
902
		if (not os.path.isfile(f'./{UElogFile}')):
903
			return -1
904
		ue_log_file = open(f'./{UElogFile}', 'r')
905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
		exitSignalReceived = False
		foundAssertion = False
		msgAssertion = ''
		msgLine = 0
		foundSegFault = False
		foundRealTimeIssue = False
		uciStatMsgCount = 0
		pdcpDataReqFailedCount = 0
		badDciCount = 0
		f1aRetransmissionCount = 0
		fatalErrorCount = 0
		macBsrTimerExpiredCount = 0
		rrcConnectionRecfgComplete = 0
		no_cell_sync_found = False
		mib_found = False
		frequency_found = False
		plmn_found = False
		nrUEFlag = False
		nrDecodeMib = 0
		nrFoundDCI = 0
		nrCRCOK = 0
		mbms_messages = 0
927 928
		nbPduSessAccept = 0
		nbPduDiscard = 0
929 930 931
		HTML.htmlUEFailureMsg=''
		global_status = CONST.ALL_PROCESSES_OK
		for line in ue_log_file.readlines():
932
			result = re.search('nr_synchro_time|Starting NR UE soft modem', str(line))
933 934 935 936 937 938 939 940 941 942 943 944
			if result is not None:
				nrUEFlag = True
			if nrUEFlag:
				result = re.search('decode mib', str(line))
				if result is not None:
					nrDecodeMib += 1
				result = re.search('found 1 DCIs', str(line))
				if result is not None:
					nrFoundDCI += 1
				result = re.search('CRC OK', str(line))
				if result is not None:
					nrCRCOK += 1
945 946 947 948 949 950
				result = re.search('Received PDU Session Establishment Accept', str(line))
				if result is not None:
					nbPduSessAccept += 1
				result = re.search('warning: discard PDU, sn out of window', str(line))
				if result is not None:
					nbPduDiscard += 1
951
				result = re.search('--nfapi STANDALONE_PNF --node-number 2 --sa', str(line))
952 953
				if result is not None:
					frequency_found = True
954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005
			result = re.search('Exiting OAI softmodem', str(line))
			if result is not None:
				exitSignalReceived = True
			result = re.search('System error|[Ss]egmentation [Ff]ault|======= Backtrace: =========|======= Memory map: ========', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Cc]ore [dD]ump', str(line))
			if result is not None and not exitSignalReceived:
				foundSegFault = True
			result = re.search('[Aa]ssertion', str(line))
			if result is not None and not exitSignalReceived:
				foundAssertion = True
			result = re.search('LLL', str(line))
			if result is not None and not exitSignalReceived:
				foundRealTimeIssue = True
			if foundAssertion and (msgLine < 3):
				msgLine += 1
				msgAssertion += str(line)
			result = re.search('uci->stat', str(line))
			if result is not None and not exitSignalReceived:
				uciStatMsgCount += 1
			result = re.search('PDCP data request failed', str(line))
			if result is not None and not exitSignalReceived:
				pdcpDataReqFailedCount += 1
			result = re.search('bad DCI 1', str(line))
			if result is not None and not exitSignalReceived:
				badDciCount += 1
			result = re.search('Format1A Retransmission but TBS are different', str(line))
			if result is not None and not exitSignalReceived:
				f1aRetransmissionCount += 1
			result = re.search('FATAL ERROR', str(line))
			if result is not None and not exitSignalReceived:
				fatalErrorCount += 1
			result = re.search('MAC BSR Triggered ReTxBSR Timer expiry', str(line))
			if result is not None and not exitSignalReceived:
				macBsrTimerExpiredCount += 1
			result = re.search('Generating RRCConnectionReconfigurationComplete', str(line))
			if result is not None:
				rrcConnectionRecfgComplete += 1
			# No cell synchronization found, abandoning
			result = re.search('No cell synchronization found, abandoning', str(line))
			if result is not None:
				no_cell_sync_found = True
			if RAN.eNBmbmsEnables[0]:
				result = re.search('TRIED TO PUSH MBMS DATA', str(line))
				if result is not None:
					mbms_messages += 1
			result = re.search("MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P<nidcell>\d{1,3}), N_RB_DL (?P<n_rb_dl>\d{1,3}), PHICH DURATION (?P<phich_duration>\d), PHICH RESOURCE (?P<phich_resource>.{1,4}), TX_ANT (?P<tx_ant>\d)", str(line))
			if result is not None and (not mib_found):
				try:
					mibMsg = "MIB Information: " + result.group(1) + ', ' + result.group(2)
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
1006
					logging.debug(f'\033[94m{mibMsg}\033[0m')
1007 1008
					mibMsg = "    nidcell = " + result.group('nidcell')
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg
1009
					logging.debug(f'\033[94m{mibMsg}\033[0m')
1010 1011
					mibMsg = "    n_rb_dl = " + result.group('n_rb_dl')
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
1012
					logging.debug(f'\033[94m{mibMsg}\033[0m')
1013 1014
					mibMsg = "    phich_duration = " + result.group('phich_duration')
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg
1015
					logging.debug(f'\033[94m{mibMsg}\033[0m')
1016 1017
					mibMsg = "    phich_resource = " + result.group('phich_resource')
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
1018
					logging.debug(f'\033[94m{mibMsg}\033[0m')
1019 1020
					mibMsg = "    tx_ant = " + result.group('tx_ant')
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
1021
					logging.debug(f'\033[94m{mibMsg}\033[0m')
1022 1023
					mib_found = True
				except Exception as e:
1024
					logging.error(f'\033[91m MIB marker was not found \033[0m')
1025
			result = re.search("Initial sync: pbch decoded sucessfully", str(line))
1026 1027
			if result is not None and (not frequency_found):
				try:
1028
					mibMsg = f"UE decoded PBCH successfully"
1029
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
1030
					logging.debug(f'\033[94m{mibMsg}\033[0m')
1031 1032
					frequency_found = True
				except Exception as e:
1033
					logging.error(f'\033[91m UE did not find PBCH\033[0m')
1034 1035 1036
			result = re.search("PLMN MCC (?P<mcc>\d{1,3}), MNC (?P<mnc>\d{1,3}), TAC", str(line))
			if result is not None and (not plmn_found):
				try:
1037
					mibMsg = f"PLMN MCC = {result.group('mcc')} MNC = {result.group('mnc')}"
1038
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
1039
					logging.debug(f'\033[94m{mibMsg}\033[0m')
1040 1041
					plmn_found = True
				except Exception as e:
1042
					logging.error(f'\033[91m PLMN not found \033[0m')
1043 1044 1045
			result = re.search("Found (?P<operator>[\w,\s]{1,15}) \(name from internal table\)", str(line))
			if result is not None:
				try:
1046
					mibMsg = f"The operator is: {result.group('operator')}"
1047
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
1048
					logging.debug(f'\033[94m{mibMsg}\033[0m')
1049
				except Exception as e:
1050
					logging.error(f'\033[91m Operator name not found \033[0m')
1051 1052 1053
			result = re.search("SIB5 InterFreqCarrierFreq element (.{1,4})/(.{1,4})", str(line))
			if result is not None:
				try:
1054
					mibMsg = f'SIB5 InterFreqCarrierFreq element {result.group(1)}/{result.group(2)}'
1055
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + ' -> '
1056
					logging.debug(f'\033[94m{mibMsg}\033[0m')
1057
				except Exception as e:
1058
					logging.error(f'\033[91m SIB5 InterFreqCarrierFreq element not found \033[0m')
1059 1060 1061 1062 1063 1064 1065
			result = re.search("DL Carrier Frequency/ARFCN : \-*(?P<carrier_frequency>\d{1,15}/\d{1,4})", str(line))
			if result is not None:
				try:
					freq = result.group('carrier_frequency')
					new_freq = re.sub('/[0-9]+','',freq)
					float_freq = float(new_freq) / 1000000
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'DL Freq: ' + ('%.1f' % float_freq) + ' MHz'
1066
					logging.debug(f'\033[94m    DL Carrier Frequency is:  {freq}\033[0m')
1067
				except Exception as e:
1068
					logging.error(f'\033[91m    DL Carrier Frequency not found \033[0m')
1069 1070 1071 1072 1073
			result = re.search("AllowedMeasBandwidth : (?P<allowed_bandwidth>\d{1,7})", str(line))
			if result is not None:
				try:
					prb = result.group('allowed_bandwidth')
					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + ' -- PRB: ' + prb + '\n'
1074
					logging.debug(f'\033[94m    AllowedMeasBandwidth: {prb}\033[0m')
1075
				except Exception as e:
1076
					logging.error(f'\033[91m    AllowedMeasBandwidth not found \033[0m')
1077 1078
		ue_log_file.close()
		if rrcConnectionRecfgComplete > 0:
1079 1080
			statMsg = f'UE connected to eNB ({rrcConnectionRecfgComplete}) RRCConnectionReconfigurationComplete message(s) generated)'
			logging.debug(f'\033[94m{statMsg}\033[0m')
1081 1082 1083
			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
		if nrUEFlag:
			if nrDecodeMib > 0:
1084 1085
				statMsg = f'UE showed {nrDecodeMib} "MIB decode" message(s)'
				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1086 1087
				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
			if nrFoundDCI > 0:
1088 1089
				statMsg = f'UE showed {nrFoundDCI} "DCI found" message(s)'
				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1090 1091
				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
			if nrCRCOK > 0:
1092 1093
				statMsg = f'UE showed {nrCRCOK} "PDSCH decoding" message(s)'
				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1094 1095 1096
				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
			if not frequency_found:
				statMsg = 'NR-UE could NOT synch!'
1097
				logging.error(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1098
				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
1099
			if nbPduSessAccept > 0:
1100 1101
				statMsg = f'UE showed {nbPduSessAccept} "Received PDU Session Establishment Accept" message(s)'
				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1102 1103
				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
			if nbPduDiscard > 0:
1104 1105
				statMsg = f'UE showed {nbPduDiscard} "warning: discard PDU, sn out of window" message(s)'
				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1106
				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
1107
		if uciStatMsgCount > 0:
1108 1109
			statMsg = f'UE showed {uciStatMsgCount} "uci->stat" message(s)'
			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1110 1111
			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
		if pdcpDataReqFailedCount > 0:
1112 1113
			statMsg = f'UE showed {pdcpDataReqFailedCount} "PDCP data request failed" message(s)'
			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1114 1115
			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
		if badDciCount > 0:
1116 1117
			statMsg = f'UE showed {badDciCount} "bad DCI 1(A)" message(s)'
			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1118 1119
			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
		if f1aRetransmissionCount > 0:
1120 1121
			statMsg = f'UE showed {f1aRetransmissionCount} "Format1A Retransmission but TBS are different" message(s)'
			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1122 1123
			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
		if fatalErrorCount > 0:
1124 1125
			statMsg = f'UE showed {fatalErrorCount} "FATAL ERROR:" message(s)'
			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1126 1127
			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
		if macBsrTimerExpiredCount > 0:
1128 1129
			statMsg = f'UE showed {fatalErrorCount} "MAC BSR Triggered ReTxBSR Timer expiry" message(s)'
			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1130 1131 1132
			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
		if RAN.eNBmbmsEnables[0]:
			if mbms_messages > 0:
1133 1134
				statMsg = f'UE showed {mbms_messages} "TRIED TO PUSH MBMS DATA" message(s)'
				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
1135 1136
			else:
				statMsg = 'UE did NOT SHOW "TRIED TO PUSH MBMS DATA" message(s)'
1137
				logging.debug(f'\u001B[1;30;41m{statMsg}\u001B[0m')
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
				global_status = CONST.OAI_UE_PROCESS_NO_MBMS_MSGS
			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
		if foundSegFault:
			logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m')
			if not nrUEFlag:
				global_status = CONST.OAI_UE_PROCESS_SEG_FAULT
			else:
				if not frequency_found:
					global_status = CONST.OAI_UE_PROCESS_SEG_FAULT
		if foundAssertion:
			logging.debug('\u001B[1;30;43m UE showed an assertion! \u001B[0m')
			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE showed an assertion!\n'
			if not nrUEFlag:
				if not mib_found or not frequency_found:
					global_status = CONST.OAI_UE_PROCESS_ASSERTION
			else:
				if not frequency_found:
					global_status = CONST.OAI_UE_PROCESS_ASSERTION
		if foundRealTimeIssue:
			logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m')
			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE faced real time issues!\n'
		if nrUEFlag:
			if not frequency_found:
				global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC
		else:
			if no_cell_sync_found and not mib_found:
				logging.debug('\u001B[1;37;41m UE could not synchronize ! \u001B[0m')
				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE could not synchronize!\n'
				global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC
		return global_status

1169
	def TerminateUE(self, HTML):
1170
		ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
1171 1172 1173 1174 1175 1176
		with concurrent.futures.ThreadPoolExecutor() as executor:
			futures = [executor.submit(ue.terminate) for ue in ues]
			archives = [f.result() for f in futures]
		archive_info = [f'Log at: {a}' if a else 'No log available' for a in archives]
		messages = [f"UE {ue.getName()}: {log}" for (ue, log) in zip(ues, archive_info)]
		HTML.CreateHtmlTestRowQueue(f'N/A', 'OK', messages)
1177

Robert Schmidt's avatar
Robert Schmidt committed
1178
	def TerminateOAIUE(self,HTML,RAN,EPC,CONTAINERS):
hardy's avatar
hardy committed
1179
		SSH = sshconnection.SSHConnection()
1180
		SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
1181
		SSH.command(f'cd {self.UESourceCodePath}/cmake_targets', '\$', 5)
1182 1183 1184
		SSH.command('ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
		result = re.search('-uesoftmodem', SSH.getBefore())
		if result is not None:
1185
			SSH.command(f'echo {self.UEPassword} | sudo -S killall --signal SIGINT -r .*-uesoftmodem || true', '\$', 5)
1186 1187 1188 1189
			time.sleep(10)
			SSH.command('ps -aux | grep --color=never softmodem | grep -v grep', '\$', 5)
			result = re.search('-uesoftmodem', SSH.getBefore())
			if result is not None:
1190
				SSH.command(f'echo {self.UEPassword} | sudo -S killall --signal SIGKILL -r .*-uesoftmodem || true', '\$', 5)
1191
				time.sleep(5)
1192
		SSH.command(f'rm -f my-lte-uesoftmodem-run {self.UE_instance}.sh', '\$', 5)
1193 1194 1195
		SSH.close()
		result = re.search('ue_', str(self.UELogFile))
		if result is not None:
1196
			copyin_res = SSH.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword,f'{self.UESourceCodePath}/cmake_targets/{self.UELogFile}', '.')
1197 1198 1199 1200 1201 1202 1203
			if (copyin_res == -1):
				logging.debug('\u001B[1;37;41m Could not copy UE logfile to analyze it! \u001B[0m')
				HTML.htmlUEFailureMsg='Could not copy UE logfile to analyze it!'
				HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OAI_UE_PROCESS_NOLOGFILE_TO_ANALYZE, 'UE')
				self.UELogFile = ''
				return
			logging.debug('\u001B[1m Analyzing UE logfile \u001B[0m')
1204
			logStatus = self.AnalyzeLogFile_UE(self.UELogFile,HTML,RAN)
1205 1206 1207 1208 1209 1210
			result = re.search('--no-L2-connect', str(self.Initialize_OAI_UE_args))
			if result is not None:
				ueAction = 'Sniffing'
			else:
				ueAction = 'Connection'
			if (logStatus < 0):
1211
				logging.debug(f'\u001B[1m {ueAction} Failed \u001B[0m')
1212 1213 1214 1215 1216 1217 1218
				HTML.htmlUEFailureMsg='<b>' + ueAction + ' Failed</b>\n' + HTML.htmlUEFailureMsg
				HTML.CreateHtmlTestRow('N/A', 'KO', logStatus, 'UE')
				if self.air_interface == 'lte-uesoftmodem':
					# In case of sniffing on commercial eNBs we have random results
					# Not an error then
					if (logStatus != CONST.OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'):
						self.Initialize_OAI_UE_args = ''
Robert Schmidt's avatar
Robert Schmidt committed
1219
						self.AutoTerminateUEandeNB(HTML,RAN,EPC,CONTAINERS)
1220 1221 1222
				else:
					if (logStatus == CONST.OAI_UE_PROCESS_COULD_NOT_SYNC):
						self.Initialize_OAI_UE_args = ''
Robert Schmidt's avatar
Robert Schmidt committed
1223
						self.AutoTerminateUEandeNB(HTML,RAN,EPC,CONTAINERS)
1224
			else:
1225
				logging.debug(f'\u001B[1m {ueAction} Completed \u001B[0m')
1226 1227 1228 1229 1230 1231
				HTML.htmlUEFailureMsg='<b>' + ueAction + ' Completed</b>\n' + HTML.htmlUEFailureMsg
				HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
			self.UELogFile = ''
		else:
			HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)

Robert Schmidt's avatar
Robert Schmidt committed
1232 1233
	def AutoTerminateUEandeNB(self,HTML,RAN,EPC,CONTAINERS):
		# TODO: terminate UE?
1234 1235
		if (self.Initialize_OAI_UE_args != ''):
			self.testCase_id = 'AUTO-KILL-OAI-UE'
1236
			HTML.testCase_id = self.testCase_id
1237
			self.desc = 'Automatic Termination of OAI-UE'
1238
			HTML.desc = self.desc
1239
			self.ShowTestID()
Robert Schmidt's avatar
Robert Schmidt committed
1240
			self.TerminateOAIUE(HTML,RAN,EPC,CONTAINERS)
1241
		if (RAN.Initialize_eNB_args != ''):
1242
			self.testCase_id = 'AUTO-KILL-RAN'
1243
			HTML.testCase_id = self.testCase_id
1244
			self.desc = 'Automatic Termination of all RAN nodes'
1245
			HTML.desc = self.desc
1246
			self.ShowTestID()
1247 1248
			#terminate all RAN nodes eNB/gNB/OCP
			for instance in range(0, len(RAN.air_interface)):
1249
				if RAN.air_interface[instance]!='':
1250
					logging.debug(f'Auto Termination of Instance {instance} : {RAN.air_interface[instance]}')
1251
					RAN.eNB_instance=instance
Raphael Defosseux's avatar
Raphael Defosseux committed
1252
					RAN.TerminateeNB(HTML,EPC)
1253 1254 1255 1256 1257 1258 1259 1260 1261
		if CONTAINERS.yamlPath[0] != '':
			self.testCase_id = 'AUTO-KILL-CONTAINERS'
			HTML.testCase_id = self.testCase_id
			self.desc = 'Automatic Termination of all RAN containers'
			HTML.desc = self.desc
			self.ShowTestID()
			for instance in range(0, len(CONTAINERS.yamlPath)):
				if CONTAINERS.yamlPath[instance]!='':
					CONTAINERS.eNB_instance=instance
1262 1263 1264 1265
					if CONTAINERS.deployKind[instance]:
						CONTAINERS.UndeployObject(HTML,RAN)
					else:
						CONTAINERS.UndeployGenObject(HTML,RAN, self)
1266 1267
		RAN.prematureExit=True

hardy's avatar
hardy committed
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
	#this function is called only if eNB/gNB fails to start
	#RH to be re-factored
	def AutoTerminateeNB(self,HTML,RAN,EPC,CONTAINERS):
		if (RAN.Initialize_eNB_args != ''):
			self.testCase_id = 'AUTO-KILL-RAN'
			HTML.testCase_id = self.testCase_id
			self.desc = 'Automatic Termination of all RAN nodes'
			HTML.desc = self.desc
			self.ShowTestID()
			#terminate all RAN nodes eNB/gNB/OCP
			for instance in range(0, len(RAN.air_interface)):
				if RAN.air_interface[instance]!='':
1280
					logging.debug(f'Auto Termination of Instance {instance} : {RAN.air_interface[instance]}')
hardy's avatar
hardy committed
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291
					RAN.eNB_instance=instance
					RAN.TerminateeNB(HTML,EPC)
		if CONTAINERS.yamlPath[0] != '':
			self.testCase_id = 'AUTO-KILL-CONTAINERS'
			HTML.testCase_id = self.testCase_id
			self.desc = 'Automatic Termination of all RAN containers'
			HTML.desc = self.desc
			self.ShowTestID()
			for instance in range(0, len(CONTAINERS.yamlPath)):
				if CONTAINERS.yamlPath[instance]!='':
					CONTAINERS.eNB_instance=instance
1292 1293 1294 1295
					if CONTAINERS.deployKind[instance]:
						CONTAINERS.UndeployObject(HTML,RAN)
					else:
						CONTAINERS.UndeployGenObject(HTML,RAN,self)
hardy's avatar
hardy committed
1296 1297
		RAN.prematureExit=True

1298 1299 1300 1301
	def IdleSleep(self,HTML):
		time.sleep(self.idle_sleep_time)
		HTML.CreateHtmlTestRow(str(self.idle_sleep_time) + ' sec', 'OK', CONST.ALL_PROCESSES_OK)

Jaroslava Fiedlerova's avatar
Jaroslava Fiedlerova committed
1302
	def X2_Status(self, idx, fileName, EPC):
1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
		cmd = "curl --silent http://" + EPC.IPAddress + ":9999/stats | jq '.' > " + fileName
		message = cmd + '\n'
		logging.debug(cmd)
		subprocess.run(cmd, shell=True)
		if idx == 0:
			cmd = "jq '.mac_stats | length' " + fileName
			strNbEnbs = subprocess.check_output(cmd, shell=True, universal_newlines=True)
			self.x2NbENBs = int(strNbEnbs.strip())
		cnt = 0
		while cnt < self.x2NbENBs:
			cmd = "jq '.mac_stats[" + str(cnt) + "].bs_id' " + fileName
			bs_id = subprocess.check_output(cmd, shell=True, universal_newlines=True)
			self.x2ENBBsIds[idx].append(bs_id.strip())
			cmd = "jq '.mac_stats[" + str(cnt) + "].ue_mac_stats | length' " + fileName
			stNbUEs = subprocess.check_output(cmd, shell=True, universal_newlines=True)
			nbUEs = int(stNbUEs.strip())
			ueIdx = 0
			self.x2ENBConnectedUEs[idx].append([])
			while ueIdx < nbUEs:
				cmd = "jq '.mac_stats[" + str(cnt) + "].ue_mac_stats[" + str(ueIdx) + "].rnti' " + fileName
				rnti = subprocess.check_output(cmd, shell=True, universal_newlines=True)
				self.x2ENBConnectedUEs[idx][cnt].append(rnti.strip())
				ueIdx += 1
			cnt += 1

		cnt = 0
		while cnt < self.x2NbENBs:
			msg = "   -- eNB: " + str(self.x2ENBBsIds[idx][cnt]) + " is connected to " + str(len(self.x2ENBConnectedUEs[idx][cnt])) + " UE(s)"
			logging.debug(msg)
			message += msg + '\n'
			ueIdx = 0
			while ueIdx < len(self.x2ENBConnectedUEs[idx][cnt]):
				msg = "      -- UE rnti: " + str(self.x2ENBConnectedUEs[idx][cnt][ueIdx])
				logging.debug(msg)
				message += msg + '\n'
				ueIdx += 1
			cnt += 1
		return message

	def Perform_X2_Handover(self,HTML,RAN,EPC):
		html_queue = SimpleQueue()
		fullMessage = '<pre style="background-color:white">'
1345
		msg = f'Doing X2 Handover w/ option {self.x2_ho_options}'
1346 1347 1348
		logging.debug(msg)
		fullMessage += msg + '\n'
		if self.x2_ho_options == 'network':
Robert Schmidt's avatar
Robert Schmidt committed
1349
			HTML.CreateHtmlTestRow('Cannot perform requested X2 Handover', 'KO', CONST.ALL_PROCESSES_OK)
1350 1351

	def LogCollectBuild(self,RAN):
1352 1353 1354 1355 1356
		# Some pipelines are using "none" IP / Credentials
		# In that case, just forget about it
		if RAN.eNBIPAddress == 'none' or self.UEIPAddress == 'none':
			sys.exit(0)

1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
		if (RAN.eNBIPAddress != '' and RAN.eNBUserName != '' and RAN.eNBPassword != ''):
			IPAddress = RAN.eNBIPAddress
			UserName = RAN.eNBUserName
			Password = RAN.eNBPassword
			SourceCodePath = RAN.eNBSourceCodePath
		elif (self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != ''):
			IPAddress = self.UEIPAddress
			UserName = self.UEUserName
			Password = self.UEPassword
			SourceCodePath = self.UESourceCodePath
		else:
			sys.exit('Insufficient Parameter')
1369
		SSH = sshconnection.SSHConnection()
1370
		SSH.open(IPAddress, UserName, Password)
1371
		SSH.command(f'cd {SourceCodePath}', '\$', 5)
1372 1373
		SSH.command('cd cmake_targets', '\$', 5)
		SSH.command('rm -f build.log.zip', '\$', 5)
1374
		SSH.command('zip -r build.log.zip build_log_*/*', '\$', 60)
1375
		SSH.close()
1376

1377
	def LogCollectPing(self,EPC):
1378 1379
		# Some pipelines are using "none" IP / Credentials
		# In that case, just forget about it
1380
		if EPC.IPAddress == 'none':
1381
			sys.exit(0)
hardy's avatar
hardy committed
1382
		SSH = sshconnection.SSHConnection()
1383
		SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
1384
		SSH.command(f'cd {EPC.SourceCodePath}', '\$', 5)
1385 1386 1387 1388 1389 1390 1391
		SSH.command('cd scripts', '\$', 5)
		SSH.command('rm -f ping.log.zip', '\$', 5)
		SSH.command('zip ping.log.zip ping*.log', '\$', 60)
		SSH.command('rm ping*.log', '\$', 5)
		SSH.close()

	def LogCollectIperf(self,EPC):
1392 1393
		# Some pipelines are using "none" IP / Credentials
		# In that case, just forget about it
1394
		if EPC.IPAddress == 'none':
1395
			sys.exit(0)
hardy's avatar
hardy committed
1396
		SSH = sshconnection.SSHConnection()
1397
		SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password)
1398
		SSH.command(f'cd {EPC.SourceCodePath}', '\$', 5)
1399 1400 1401 1402 1403 1404 1405
		SSH.command('cd scripts', '\$', 5)
		SSH.command('rm -f iperf.log.zip', '\$', 5)
		SSH.command('zip iperf.log.zip iperf*.log', '\$', 60)
		SSH.command('rm iperf*.log', '\$', 5)
		SSH.close()
	
	def LogCollectOAIUE(self):
1406 1407 1408 1409
		# Some pipelines are using "none" IP / Credentials
		# In that case, just forget about it
		if self.UEIPAddress == 'none':
			sys.exit(0)
hardy's avatar
hardy committed
1410
		SSH = sshconnection.SSHConnection()
1411
		SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
1412 1413 1414 1415 1416
		SSH.command(f'cd {self.UESourceCodePath}', '\$', 5)
		SSH.command(f'cd cmake_targets', '\$', 5)
		SSH.command(f'echo {self.UEPassword} | sudo -S rm -f ue.log.zip', '\$', 5)
		SSH.command(f'echo {self.UEPassword} | sudo -S zip ue.log.zip ue*.log core* ue_*record.raw ue_*.pcap ue_*txt', '\$', 60)
		SSH.command(f'echo {self.UEPassword} | sudo -S rm ue*.log core* ue_*record.raw ue_*.pcap ue_*txt', '\$', 5)
1417 1418
		SSH.close()

1419 1420 1421 1422 1423 1424
	def ConditionalExit(self):
		if self.testUnstable:
			if self.testStabilityPointReached or self.testMinStableId == '999999':
				sys.exit(0)
		sys.exit(1)

1425
	def ShowTestID(self):
1426 1427 1428 1429
		logging.info(f'\u001B[1m----------------------------------------\u001B[0m')
		logging.info(f'\u001B[1m Test ID: {self.testCase_id} \u001B[0m')
		logging.info(f'\u001B[1m {self.desc} \u001B[0m')
		logging.info(f'\u001B[1m----------------------------------------\u001B[0m')