cls_containerize.py 67.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import sys              # arg
import re               # reg
import logging
import os
38
import shutil
39
import subprocess
40 41
import time
from multiprocessing import Process, Lock, SimpleQueue
42
from zipfile import ZipFile
43 44 45 46 47 48 49 50

#-----------------------------------------------------------
# OAI Testing modules
#-----------------------------------------------------------
import sshconnection as SSH
import helpreadme as HELP
import constants as CONST

51 52 53 54
#-----------------------------------------------------------
# Helper functions used here and in other classes
# (e.g., cls_cluster.py)
#-----------------------------------------------------------
55
def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTargetBranch, ranAllowMerge):
56 57 58 59 60
	if ranCommitID == '':
		logging.error('need ranCommitID in CreateWorkspace()')
		sys.exit('Insufficient Parameter in CreateWorkspace()')

	sshSession.command(f'rm -rf {sourcePath}', '\$', 10)
61 62
	sshSession.command('mkdir -p ' + sourcePath, '\$', 5)
	sshSession.command('cd ' + sourcePath, '\$', 5)
Robert Schmidt's avatar
Robert Schmidt committed
63 64 65 66
	# Recent version of git (>2.20?) should handle missing .git extension # without problems
	sshSession.command(f'git clone --filter=blob:none -n -b develop {ranRepository} .', '\$', 60)
	if sshSession.getBefore().count('error') > 0 or sshSession.getBefore().count('error') > 0:
		sys.exit('error during clone')
67 68 69 70 71
	sshSession.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
	sshSession.command('git config user.name "OAI Jenkins"', '\$', 5)

	sshSession.command('mkdir -p cmake_targets/log', '\$', 5)
	# if the commit ID is provided use it to point to it
72 73 74 75 76 77
	sshSession.command(f'git checkout -f {ranCommitID}', '\$', 30)
	if sshSession.getBefore().count(f'HEAD is now at {ranCommitID[:6]}') != 1:
		sshSession.command('git log --oneline | head -n5', '\$', 5)
		logging.warning(f'problems during checkout, is at: {sshSession.getBefore()}')
	else:
		logging.debug('successful checkout')
78 79 80 81
	# if the branch is not develop, then it is a merge request and we need to do
	# the potential merge. Note that merge conflicts should already been checked earlier
	if ranAllowMerge:
		if ranTargetBranch == '':
82 83 84
			ranTargetBranch = 'develop'
		logging.debug(f'Merging with the target branch: {ranTargetBranch}')
		sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 5)
85

86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
def CopyLogsToExecutor(sshSession, sourcePath, log_name, scpIp, scpUser, scpPw):
	sshSession.command(f'cd {sourcePath}/cmake_targets', '\$', 5)
	sshSession.command(f'rm -f {log_name}.zip', '\$', 5)
	sshSession.command(f'mkdir -p {log_name}', '\$', 5)
	sshSession.command(f'mv log/* {log_name}', '\$', 5)
	sshSession.command(f'zip -r -qq {log_name}.zip {log_name}', '\$', 5)

	# copy zip to executor for analysis
	if (os.path.isfile(f'./{log_name}.zip')):
		os.remove(f'./{log_name}.zip')
	if (os.path.isdir(f'./{log_name}')):
		shutil.rmtree(f'./{log_name}')
	sshSession.copyin(scpIp, scpUser, scpPw, f'{sourcePath}/cmake_targets/{log_name}.zip', '.')
	sshSession.command(f'rm -f {log_name}.zip','\$', 5)
	ZipFile(f'{log_name}.zip').extractall('.')

102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
def AnalyzeBuildLogs(buildRoot, images, globalStatus):
	collectInfo = {}
	for image in images:
		files = {}
		file_list = [f for f in os.listdir(f'{buildRoot}/{image}') if os.path.isfile(os.path.join(f'{buildRoot}/{image}', f)) and f.endswith('.txt')]
		# Analyze the "sub-logs" of every target image
		for fil in file_list:
			errorandwarnings = {}
			warningsNo = 0
			errorsNo = 0
			with open(f'{buildRoot}/{image}/{fil}', mode='r') as inputfile:
				for line in inputfile:
					result = re.search(' ERROR ', str(line))
					if result is not None:
						errorsNo += 1
					result = re.search(' error:', str(line))
					if result is not None:
						errorsNo += 1
					result = re.search(' WARNING ', str(line))
					if result is not None:
						warningsNo += 1
					result = re.search(' warning:', str(line))
					if result is not None:
						warningsNo += 1
				errorandwarnings['errors'] = errorsNo
				errorandwarnings['warnings'] = warningsNo
				errorandwarnings['status'] = globalStatus
			files[fil] = errorandwarnings
		# Analyze the target image
		if os.path.isfile(f'{buildRoot}/{image}.log'):
			errorandwarnings = {}
			committed = False
			tagged = False
			with open(f'{buildRoot}/{image}.log', mode='r') as inputfile:
				startOfTargetImageCreation = False # check for tagged/committed only after image created
				for line in inputfile:
					result = re.search(f'FROM .* [aA][sS] {image}$', str(line))
					if result is not None:
						startOfTargetImageCreation = True
					if startOfTargetImageCreation:
						lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None
						tagged = tagged or lineHasTag
						# the OpenShift Cluster builder prepends image registry URL
						lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None
						committed = committed or lineHasCommit
			errorandwarnings['errors'] = 0 if committed or tagged else 1
			errorandwarnings['warnings'] = 0
			errorandwarnings['status'] = committed or tagged
			files['Target Image Creation'] = errorandwarnings
		collectInfo[image] = files
	return collectInfo

154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class Containerize():

	def __init__(self):
		
		self.ranRepository = ''
		self.ranBranch = ''
		self.ranAllowMerge = False
		self.ranCommitID = ''
		self.ranTargetBranch = ''
		self.eNBIPAddress = ''
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.eNB1IPAddress = ''
		self.eNB1UserName = ''
		self.eNB1Password = ''
		self.eNB1SourceCodePath = ''
		self.eNB2IPAddress = ''
		self.eNB2UserName = ''
		self.eNB2Password = ''
		self.eNB2SourceCodePath = ''
		self.forcedWorkspaceCleanup = False
		self.imageKind = ''
180
		self.proxyCommit = None
181 182
		self.eNB_instance = 0
		self.eNB_serverId = ['', '', '']
183
		self.yamlPath = ['', '', '']
184 185 186
		self.services = ['', '', '']
		self.nb_healthy = [0, 0, 0]
		self.exitStatus = 0
187 188
		self.eNB_logFile = ['', '', '']

189
		self.testCase_id = ''
190 191 192

		self.flexranCtrlDeployed = False
		self.flexranCtrlIpAddress = ''
193
		self.cli = ''
194
		self.cliBuildOptions = ''
195 196
		self.dockerfileprefix = ''
		self.host = ''
197

198
		self.deployedContainers = []
199
		self.tsharkStarted = False
200 201 202 203 204 205 206 207
		self.pingContName = ''
		self.pingOptions = ''
		self.pingLossThreshold = ''
		self.svrContName = ''
		self.svrOptions = ''
		self.cliContName = ''
		self.cliOptions = ''

208 209 210
		self.imageToCopy = ''
		self.registrySvrId = ''
		self.testSvrId = ''
211
		self.imageToPull = []
212

213 214 215
		#checkers from xml
		self.ran_checkers={}

216 217 218 219
#-----------------------------------------------------------
# Container management functions
#-----------------------------------------------------------

Raphael Defosseux's avatar
Raphael Defosseux committed
220
	def BuildImage(self, HTML):
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
		if self.eNB_serverId[self.eNB_instance] == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId[self.eNB_instance] == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId[self.eNB_instance] == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
		logging.debug('Building on server: ' + lIpAddr)
		mySSH = SSH.SSHConnection()
		mySSH.open(lIpAddr, lUserName, lPassWord)
245 246 247 248 249 250 251 252
	
		# Checking the hostname to get adapted on cli and dockerfileprefixes
		mySSH.command('hostnamectl', '\$', 5)
		result = re.search('Ubuntu|Red Hat',  mySSH.getBefore())
		self.host = result.group(0)
		if self.host == 'Ubuntu':
			self.cli = 'docker'
			self.dockerfileprefix = '.ubuntu18'
253
			self.cliBuildOptions = '--no-cache'
254
		elif self.host == 'Red Hat':
255
			self.cli = 'sudo podman'
256
			self.dockerfileprefix = '.rhel8.2'
257
			self.cliBuildOptions = '--no-cache --disable-compression'
258

259 260
                # we always build the ran-build image with all targets
		imageNames = [('ran-build', 'build')]
261
		result = re.search('eNB', self.imageKind)
262
		# Creating a tupple with the imageName and the DockerFile prefix pattern on obelix
263 264 265 266 267 268 269 270 271
		if result is not None:
			imageNames.append(('oai-enb', 'eNB'))
		else:
			result = re.search('gNB', self.imageKind)
			if result is not None:
				imageNames.append(('oai-gnb', 'gNB'))
			else:
				result = re.search('all', self.imageKind)
				if result is not None:
272
					imageNames.append(('oai-enb', 'eNB'))
273
					imageNames.append(('oai-gnb', 'gNB'))
274
					imageNames.append(('oai-lte-ue', 'lteUE'))
275
					imageNames.append(('oai-nr-ue', 'nrUE'))
276 277
					if self.host == 'Red Hat':
						imageNames.append(('oai-physim', 'phySim'))
278 279
					if self.host == 'Ubuntu':
						imageNames.append(('oai-lte-ru', 'lteRU'))
280
		
281 282 283
		# Workaround for some servers, we need to erase completely the workspace
		if self.forcedWorkspaceCleanup:
			mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
284
	
Raphael Defosseux's avatar
Raphael Defosseux committed
285
		self.testCase_id = HTML.testCase_id
286
	
287
		CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
288

289 290
 		# if asterix, copy the entitlement and subscription manager configurations
		if self.host == 'Red Hat':
291 292 293 294
			mySSH.command('mkdir -p ./etc-pki-entitlement ./rhsm-conf ./rhsm-ca', '\$', 5)
			mySSH.command('sudo cp /etc/rhsm/rhsm.conf ./rhsm-conf/', '\$', 5)
			mySSH.command('sudo cp /etc/rhsm/ca/redhat-uep.pem ./rhsm-ca/', '\$', 5)
			mySSH.command('sudo cp /etc/pki/entitlement/*.pem ./etc-pki-entitlement/', '\$', 5)
295

296 297 298
		baseImage = 'ran-base'
		baseTag = 'develop'
		forceBaseImageBuild = False
299
		imageTag = 'develop'
300 301
		if (self.ranAllowMerge):
			imageTag = 'ci-temp'
302
			if self.ranTargetBranch == 'develop':
303
				mySSH.command('git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5)
304 305
				result = re.search('index', mySSH.getBefore())
				if result is not None:
306 307
					forceBaseImageBuild = True
					baseTag = 'ci-temp'
308
		else:
309
			forceBaseImageBuild = True
310

311
		# Let's remove any previous run artifacts if still there
312
		mySSH.command(self.cli + ' image prune --force', '\$', 30)
313 314
		if forceBaseImageBuild:
			mySSH.command(self.cli + ' image rm ' + baseImage + ':' + baseTag + ' || true', '\$', 30)
315
		for image,pattern in imageNames:
316
			mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag + ' || true', '\$', 30)
317

318 319 320
		# Build the base image only on Push Events (not on Merge Requests)
		# On when the base image docker file is being modified.
		if forceBaseImageBuild:
321
			mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + baseImage + ' --tag ' + baseImage + ':' + baseTag + ' --file docker/Dockerfile.base' + self.dockerfileprefix + ' . > cmake_targets/log/ran-base.log 2>&1', '\$', 1600)
322 323
		# First verify if the base image was properly created.
		mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + baseImage + ':' + baseTag, '\$', 5)
324
		allImagesSize = {}
325
		if mySSH.getBefore().count('o such image') != 0:
326
			logging.error('\u001B[1m Could not build properly ran-base\u001B[0m')
327
			# Recover the name of the failed container?
328
			mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5)
329
			mySSH.command(self.cli + ' image prune --force', '\$', 30)
330 331 332 333 334
			mySSH.close()
			logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
			HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
			HTML.CreateHtmlTabFooter(False)
			sys.exit(1)
335
		else:
336 337 338 339 340 341 342 343 344 345 346 347 348 349
			result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
			if result is not None:
				size = float(result.group("size")) / 1000000
				imageSizeStr = f'{size:.1f}'
				logging.debug(f'\u001B[1m   ran-base size is {imageSizeStr} Mbytes\u001B[0m')
				allImagesSize['ran-base'] = f'{imageSizeStr} Mbytes'
			else:
				logging.debug('ran-base size is unknown')

		# Recover build logs, for the moment only possible when build is successful
		mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5)
		mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5)
		mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5)
		mySSH.command(self.cli + ' rm -f test', '\$', 5)
350 351

		# Build the target image(s)
352 353
		status = True
		attemptedImages = ['ran-base']
354
		for image,pattern in imageNames:
355
			attemptedImages += [image]
356
			# the archived Dockerfiles have "ran-base:latest" as base image
357
			# we need to update them with proper tag
358
			mySSH.command(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5)
359
			if image != 'ran-build':
360 361
				mySSH.command(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5)
			mySSH.command(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', '\$', 1200)
362 363 364
			# split the log
			mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5)
			mySSH.command('python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/' + image + '.log', '\$', 5)
365 366
			# checking the status of the build
			mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5)
367 368
			if mySSH.getBefore().count('o such image') != 0:
				logging.error('\u001B[1m Could not build properly ' + image + '\u001B[0m')
369
				status = False
370
				# Here we should check if the last container corresponds to a failed command and destroy it
371
				mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5)
372 373
				allImagesSize[image] = 'N/A -- Build Failed'
				break
374
			else:
375
				result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
376
				if result is not None:
377 378 379 380
					size = float(result.group("size")) / 1000000
					imageSizeStr = f'{size:.1f}'
					logging.debug(f'\u001B[1m   {image} size is {imageSizeStr} Mbytes\u001B[0m')
					allImagesSize[image] = f'{imageSizeStr} Mbytes'
381
				else:
382 383
					logging.debug(f'{image} size is unknown')
					allImagesSize[image] = 'unknown'
384 385
			# Now pruning dangling images in between target builds
			mySSH.command(self.cli + ' image prune --force', '\$', 30)
386

387
		# Remove all intermediate build images and clean up
388
		if self.ranAllowMerge and forceBaseImageBuild:
389 390 391 392 393 394 395
			mySSH.command(f'{self.cli} image rm {baseImage}:{baseTag} || true', '\$', 30)
		mySSH.command(f'{self.cli} image rm ran-build:{imageTag}','\$', 30)
		mySSH.command(f'{self.cli} volume prune --force','\$', 15)

		# create a zip with all logs
		build_log_name = f'build_log_{self.testCase_id}'
		CopyLogsToExecutor(mySSH, lSourcePath, build_log_name, lIpAddr, lUserName, lPassWord)
396
		mySSH.close()
397

398
		# Analyze the logs
399
		collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status)
400
		
401 402 403
		if status:
			logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
			HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK)
404
			HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
405 406 407
		else:
			logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
			HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
408
			HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
409 410
			HTML.CreateHtmlTabFooter(False)
			sys.exit(1)
411

412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462
	def BuildProxy(self, HTML):
		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
		if self.eNB_serverId[self.eNB_instance] == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId[self.eNB_instance] == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId[self.eNB_instance] == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
		if self.proxyCommit is None:
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter (need proxyCommit for proxy build)')
		logging.debug('Building on server: ' + lIpAddr)
		mySSH = SSH.SSHConnection()
		mySSH.open(lIpAddr, lUserName, lPassWord)

		# Check that we are on Ubuntu
		mySSH.command('hostnamectl', '\$', 5)
		result = re.search('Ubuntu',  mySSH.getBefore())
		self.host = result.group(0)
		if self.host != 'Ubuntu':
			logging.error('\u001B[1m Can build proxy only on Ubuntu server\u001B[0m')
			mySSH.close()
			sys.exit(1)

		self.cli = 'docker'
		self.cliBuildOptions = '--no-cache'

		# Workaround for some servers, we need to erase completely the workspace
		if self.forcedWorkspaceCleanup:
			mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)

		oldRanCommidID = self.ranCommitID
		oldRanRepository = self.ranRepository
		oldRanAllowMerge = self.ranAllowMerge
		self.ranCommitID = self.proxyCommit
		self.ranRepository = 'https://github.com/EpiSci/oai-lte-5g-multi-ue-proxy.git'
		self.ranAllowMerge = False
463
		CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
		# to prevent accidentally overwriting data that might be used later
		self.ranCommitID = oldRanCommidID
		self.ranRepository = oldRanRepository
		self.ranAllowMerge = oldRanAllowMerge

		# Let's remove any previous run artifacts if still there
		mySSH.command(self.cli + ' image prune --force', '\$', 30)
		# Remove any previous proxy image
		mySSH.command(self.cli + ' image rm oai-lte-multi-ue-proxy:latest || true', '\$', 30)

		tag = self.proxyCommit
		logging.debug('building L2sim proxy image for tag ' + tag)
		# check if the corresponding proxy image with tag exists. If not, build it
		mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5)
		buildProxy = mySSH.getBefore().count('o such image') != 0
		if buildProxy:
			mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target oai-lte-multi-ue-proxy --tag proxy:' + tag + ' --file docker/Dockerfile.ubuntu18.04 . > cmake_targets/log/proxy-build.log 2>&1', '\$', 180)
			# Note: at this point, OAI images are flattened, but we cannot do this
			# here, as the flatten script is not in the proxy repo
			mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5)
			if mySSH.getBefore().count('o such image') != 0:
				logging.error('\u001B[1m Build of L2sim proxy failed\u001B[0m')
				mySSH.close()
				HTML.CreateHtmlTestRow('commit ' + tag, 'KO', CONST.ALL_PROCESSES_OK)
				HTML.CreateHtmlTabFooter(False)
				sys.exit(1)
		else:
			logging.debug('L2sim proxy image for tag ' + tag + ' already exists, skipping build')

		# retag the build images to that we pick it up later
		mySSH.command('docker image tag proxy:' + tag + ' oai-lte-multi-ue-proxy:latest', '\$', 5)

		# no merge: is a push to develop, tag the image so we can push it to the registry
		if not self.ranAllowMerge:
			mySSH.command('docker image tag proxy:' + tag + ' proxy:develop', '\$', 5)

		# we assume that the host on which this is built will also run the proxy. The proxy
		# currently requires the following command, and the docker-compose up mechanism of
		# the CI does not allow to run arbitrary commands. Note that the following actually
		# belongs to the deployment, not the build of the proxy...
		logging.warning('the following command belongs to deployment, but no mechanism exists to exec it there!')
		mySSH.command('sudo ifconfig lo: 127.0.0.2 netmask 255.0.0.0 up', '\$', 5)

		# Analyzing the logs
		if buildProxy:
			self.testCase_id = HTML.testCase_id
			mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
			mySSH.command('mkdir -p proxy_build_log_' + self.testCase_id, '\$', 5)
			mySSH.command('mv log/* ' + 'proxy_build_log_' + self.testCase_id, '\$', 5)
			if (os.path.isfile('./proxy_build_log_' + self.testCase_id + '.zip')):
				os.remove('./proxy_build_log_' + self.testCase_id + '.zip')
			if (os.path.isdir('./proxy_build_log_' + self.testCase_id)):
				shutil.rmtree('./proxy_build_log_' + self.testCase_id)
			mySSH.command('zip -r -qq proxy_build_log_' + self.testCase_id + '.zip proxy_build_log_' + self.testCase_id, '\$', 5)
			mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/build_log_' + self.testCase_id + '.zip', '.')
			# don't delete such that we might recover the zips
			#mySSH.command('rm -f build_log_' + self.testCase_id + '.zip','\$', 5)

522 523 524 525 526 527 528 529
		# we do not analyze the logs (we assume the proxy builds fine at this stage),
		# but need to have the following information to correctly display the HTML
		files = {}
		errorandwarnings = {}
		errorandwarnings['errors'] = 0
		errorandwarnings['warnings'] = 0
		errorandwarnings['status'] = True
		files['Target Image Creation'] = errorandwarnings
530 531
		collectInfo = {}
		collectInfo['proxy'] = files
532 533
		mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5)
		result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
534
		allImagesSize = {}
535 536 537
		if result is not None:
			imageSize = float(result.group('size')) / 1000000
			logging.debug('\u001B[1m   proxy size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
538
			allImagesSize['proxy'] = str(round(imageSize,1)) + ' Mbytes'
539 540
		else:
			logging.debug('proxy size is unknown')
541
			allImagesSize['proxy'] = 'unknown'
542

543 544 545 546 547 548
		# Cleaning any created tmp volume
		mySSH.command(self.cli + ' volume prune --force || true','\$', 15)
		mySSH.close()

		logging.info('\u001B[1m Building L2sim Proxy Image Pass\u001B[0m')
		HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK)
549
		HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
550

551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584
	def ImageTagToUse(self, imageName):
		shortCommit = self.ranCommitID[0:8]
		if self.ranAllowMerge:
			tagToUse = f'{self.ranBranch}-{shortCommit}'
		else:
			tagToUse = f'develop-{shortCommit}'
		fullTag = f'porcepix.sboai.cs.eurecom.fr/{imageName}:{tagToUse}'
		return fullTag

	def Push_Image_to_Local_Registry(self, HTML):
		if self.registrySvrId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.registrySvrId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.registrySvrId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
		logging.debug('Pushing images from server: ' + lIpAddr)
		mySSH = SSH.SSHConnection()
		mySSH.open(lIpAddr, lUserName, lPassWord)

		mySSH.command('echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5)
		if re.search('Login Succeeded', mySSH.getBefore()) is None:
585 586
			msg = 'Could not log into local registry'
			logging.error(msg)
587
			mySSH.close()
588
			HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
589 590 591 592 593 594 595 596 597 598 599 600
			return False

		orgTag = 'develop'
		if self.ranAllowMerge:
			orgTag = 'ci-temp'
		imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru']
		for image in imageNames:
			tagToUse = self.ImageTagToUse(image)
			mySSH.command(f'docker image tag {image}:{orgTag} {tagToUse}', '\$', 5)
			mySSH.command(f'docker push {tagToUse}', '\$', 120)
			if re.search(': digest:', mySSH.getBefore()) is None:
				logging.debug(mySSH.getBefore())
601 602
				msg = f'Could not push {image} to local registry : {tagToUse}'
				logging.error(msg)
603
				mySSH.close()
604
				HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
605
				return False
606 607
			# TODO: once we are done migrating all types of tests, rmi also {image}:{orgTag}
			#       so when we pull on builder as a test server, it will really pull
608 609 610 611
			mySSH.command(f'docker rmi {tagToUse}', '\$', 5)

		mySSH.command('docker logout porcepix.sboai.cs.eurecom.fr', '\$', 5)
		if re.search('Removing login credentials', mySSH.getBefore()) is None:
612 613
			msg = 'Could not log off from local registry'
			logging.error(msg)
614
			mySSH.close()
615
			HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
616 617 618
			return False

		mySSH.close()
619
		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
620 621 622
		return True

	def Pull_Image_from_Local_Registry(self, HTML):
623 624
		# This method can be called either onto a remote server (different from python executor)
		# or directly on the python executor (ie lIpAddr == 'none')
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
		if self.testSvrId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.testSvrId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.testSvrId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
643 644 645 646 647 648
		if lIpAddr != 'none':
			logging.debug('Pulling images onto server: ' + lIpAddr)
			mySSH = SSH.SSHConnection()
			mySSH.open(lIpAddr, lUserName, lPassWord)
		else:
			logging.debug('Pulling images locally')
649

650 651 652 653 654 655 656 657 658 659 660 661 662
		cmd = 'echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr'
		if lIpAddr != 'none':
			mySSH.command(cmd, '\$', 5)
			response = mySSH.getBefore()
		else:
			logging.info(cmd)
			response = subprocess.check_output(cmd, shell=True, universal_newlines=True, stderr=subprocess.STDOUT)
		if re.search('Login Succeeded', response) is None:
			msg = 'Could not log into local registry'
			logging.error(msg)
			if lIpAddr != 'none':
				mySSH.close()
			HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
663 664 665 666
			return False

		for image in self.imageToPull:
			tagToUse = self.ImageTagToUse(image)
667 668 669 670 671 672 673 674 675 676 677 678 679 680
			cmd = f'docker pull {tagToUse}'
			if lIpAddr != 'none':
				mySSH.command(cmd, '\$', 120)
				response = mySSH.getBefore()
			else:
				logging.info(cmd)
				response = subprocess.check_output(cmd, shell=True, universal_newlines=True)
			if re.search('Status: Downloaded newer image for |Status: Image is up to date for', response) is None:
				logging.debug(response)
				msg = f'Could not pull {image} from local registry : {tagToUse}'
				logging.error(msg)
				if lIpAddr != 'none':
					mySSH.close()
				HTML.CreateHtmlTestRow('msg', 'KO', CONST.ALL_PROCESSES_OK)
681 682
				return False

683 684 685 686 687 688 689 690 691 692
		cmd = 'docker logout porcepix.sboai.cs.eurecom.fr'
		if lIpAddr != 'none':
			mySSH.command(cmd, '\$', 5)
			response = mySSH.getBefore()
		else:
			logging.info(cmd)
			response = subprocess.check_output(cmd, shell=True, universal_newlines=True)
		if re.search('Removing login credentials', response) is None:
			msg = 'Could not log off from local registry'
			logging.error(msg)
693
			mySSH.close()
694
			HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
695 696
			return False

697 698
		if lIpAddr != 'none':
			mySSH.close()
699
		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
700 701 702
		return True

	def Clean_Test_Server_Images(self, HTML):
703 704
		# This method can be called either onto a remote server (different from python executor)
		# or directly on the python executor (ie lIpAddr == 'none')
705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722
		if self.testSvrId == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.testSvrId == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.testSvrId == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
723 724 725 726 727 728
		if lIpAddr != 'none':
			logging.debug('Removing test images from server: ' + lIpAddr)
			mySSH = SSH.SSHConnection()
			mySSH.open(lIpAddr, lUserName, lPassWord)
		else:
			logging.debug('Removing test images locally')
729 730 731

		imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru']
		for image in imageNames:
732 733 734 735 736 737
			cmd = f'docker rmi {self.ImageTagToUse(image)} || true'
			if lIpAddr != 'none':
				mySSH.command(cmd, '\$', 5)
			else:
				logging.info(cmd)
				subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
738

739 740
		if lIpAddr != 'none':
			mySSH.close()
741
		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
742 743
		return True

Raphael Defosseux's avatar
Raphael Defosseux committed
744
	def DeployObject(self, HTML, EPC):
745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763
		if self.eNB_serverId[self.eNB_instance] == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId[self.eNB_instance] == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId[self.eNB_instance] == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
		logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m')
764

765 766 767
		mySSH = SSH.SSHConnection()
		mySSH.open(lIpAddr, lUserName, lPassWord)
		
768
		CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
769

770 771
		mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
		mySSH.command('cp docker-compose.yml ci-docker-compose.yml', '\$', 5)
772 773 774 775
		imagesList = ['oai-enb', 'oai-gnb']
		for image in imagesList:
			imageToUse = self.ImageTagToUse(image)
			mySSH.command(f'sed -i -e "s#image: {image}:latest#image: {imageToUse}#" ci-docker-compose.yml', '\$', 2)
Raphael Defosseux's avatar
Raphael Defosseux committed
776 777
		localMmeIpAddr = EPC.MmeIPAddress
		mySSH.command('sed -i -e "s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/" ci-docker-compose.yml', '\$', 2)
778 779 780 781 782 783
#		if self.flexranCtrlDeployed:
#			mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*/FLEXRAN_ENABLED: \'yes\'/" ci-docker-compose.yml', '\$', 2)
#			mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/" ci-docker-compose.yml', '\$', 2)
#		else:
#			mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*$/FLEXRAN_ENABLED: \'no\'/" ci-docker-compose.yml', '\$', 2)
#			mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/127.0.0.1/" ci-docker-compose.yml', '\$', 2)
784
		# Currently support only one
785
		mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 15)
786
		mySSH.command('docker-compose --file ci-docker-compose.yml config --services | sed -e "s@^@service=@" 2>&1', '\$', 10)
787 788 789
		result = re.search('service=(?P<svc_name>[a-zA-Z0-9\_]+)', mySSH.getBefore())
		if result is not None:
			svcName = result.group('svc_name')
790
			mySSH.command('docker-compose --file ci-docker-compose.yml up -d ' + svcName, '\$', 15)
791 792 793 794 795 796 797 798 799 800 801 802 803

		# Checking Status
		mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5)
		result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
		unhealthyNb = 0
		healthyNb = 0
		startingNb = 0
		containerName = ''
		if result is not None:
			containerName = result.group('container_name')
			time.sleep(5)
			cnt = 0
			while (cnt < 3):
804
				mySSH.command('docker inspect --format="{{.State.Health.Status}}" ' + containerName, '\$', 5)
805 806 807 808 809 810 811 812 813 814 815 816
				unhealthyNb = mySSH.getBefore().count('unhealthy')
				healthyNb = mySSH.getBefore().count('healthy') - unhealthyNb
				startingNb = mySSH.getBefore().count('starting')
				if healthyNb == 1:
					cnt = 10
				else:
					time.sleep(10)
					cnt += 1
		logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)')
		logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)')
		logging.debug(' -- ' + str(startingNb) + ' still starting container(s)')

817 818 819
		self.testCase_id = HTML.testCase_id
		self.eNB_logFile[self.eNB_instance] = 'enb_' + self.testCase_id + '.log'

820 821 822 823 824 825 826 827 828 829 830 831 832 833
		status = False
		if healthyNb == 1:
			cnt = 0
			while (cnt < 20):
				mySSH.command('docker logs ' + containerName + ' | egrep --text --color=never -i "wait|sync|Starting"', '\$', 30) 
				result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore())
				if result is None:
					time.sleep(6)
					cnt += 1
				else:
					cnt = 100
					status = True
					logging.info('\u001B[1m Deploying OAI object Pass\u001B[0m')
					time.sleep(10)
834 835 836
		else:
			# containers are unhealthy, so we won't start. However, logs are stored at the end
			# in UndeployObject so we here store the logs of the unhealthy container to report it
837 838 839
			logfilename = f'{lSourcePath}/cmake_targets/{self.eNB_logFile[self.eNB_instance]}'
			mySSH.command('docker logs {containerName} > {logfilename}', '\$', 30)
			mySSH.copyin(lIpAddr, lUserName, lPassWord, logfilename, '.')
840 841
		mySSH.close()

Raphael Defosseux's avatar
Raphael Defosseux committed
842 843 844
		if status:
			HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
		else:
845
			self.exitStatus = 1
Raphael Defosseux's avatar
Raphael Defosseux committed
846
			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK)
847

848

Raphael Defosseux's avatar
Raphael Defosseux committed
849
	def UndeployObject(self, HTML, RAN):
850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
		if self.eNB_serverId[self.eNB_instance] == '0':
			lIpAddr = self.eNBIPAddress
			lUserName = self.eNBUserName
			lPassWord = self.eNBPassword
			lSourcePath = self.eNBSourceCodePath
		elif self.eNB_serverId[self.eNB_instance] == '1':
			lIpAddr = self.eNB1IPAddress
			lUserName = self.eNB1UserName
			lPassWord = self.eNB1Password
			lSourcePath = self.eNB1SourceCodePath
		elif self.eNB_serverId[self.eNB_instance] == '2':
			lIpAddr = self.eNB2IPAddress
			lUserName = self.eNB2UserName
			lPassWord = self.eNB2Password
			lSourcePath = self.eNB2SourceCodePath
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
868
		logging.debug('\u001B[1m Undeploying OAI Object from server: ' + lIpAddr + '\u001B[0m')
869 870 871 872 873
		mySSH = SSH.SSHConnection()
		mySSH.open(lIpAddr, lUserName, lPassWord)
		mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
		# Currently support only one
		mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5)
874 875
		containerName = ''
		containerToKill = False
876
		result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
877 878
		if self.eNB_logFile[self.eNB_instance] == '':
			self.eNB_logFile[self.eNB_instance] = 'enb_' + HTML.testCase_id + '.log'
879 880
		if result is not None:
			containerName = result.group('container_name')
881 882 883 884 885 886 887
			containerToKill = True
		if containerToKill:
			mySSH.command('docker inspect ' + containerName, '\$', 30)
			result = re.search('Error: No such object: ' + containerName, mySSH.getBefore())
			if result is not None:
				containerToKill = False
		if containerToKill:
888
			mySSH.command('docker kill --signal INT ' + containerName, '\$', 30)
889 890
			time.sleep(5)
			mySSH.command('docker kill --signal KILL ' + containerName, '\$', 30)
891 892 893
			time.sleep(5)
			mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30)
			mySSH.command('docker rm -f ' + containerName, '\$', 30)
894 895
		# Forcing the down now to remove the networks and any artifacts
		mySSH.command('docker-compose --file ci-docker-compose.yml down', '\$', 5)
896 897
		# Cleaning any created tmp volume
		mySSH.command('docker volume prune --force || true', '\$', 20)
898 899 900 901

		mySSH.close()

		# Analyzing log file!
902 903 904 905
		if containerToKill:
			copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '.')
		else:
			copyin_res = 0
Raphael Defosseux's avatar
Raphael Defosseux committed
906 907 908 909 910
		nodeB_prefix = 'e'
		if (copyin_res == -1):
			HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
		else:
911 912
			if containerToKill:
				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance])
913
				logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML, self.ran_checkers)
914 915
			else:
				logStatus = 0
Raphael Defosseux's avatar
Raphael Defosseux committed
916 917
			if (logStatus < 0):
				HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
918
			else:
Raphael Defosseux's avatar
Raphael Defosseux committed
919
				HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
920
			# all the xNB run logs shall be on the server 0 for logCollecting
921
			if containerToKill and self.eNB_serverId[self.eNB_instance] != '0':
922
				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + self.eNB_logFile[self.eNB_instance], self.eNBSourceCodePath + '/cmake_targets/')
923 924
		logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')

925
	def DeployGenObject(self, HTML, RAN, UE):
926
		self.exitStatus = 0
927
		logging.debug('\u001B[1m Checking Services to deploy\u001B[0m')
928
		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose config --services'
929
		logging.info(cmd)
930 931 932 933 934 935
		try:
			listServices = subprocess.check_output(cmd, shell=True, universal_newlines=True)
		except Exception as e:
			self.exitStatus = 1
			HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK)
			return
936
		displayUsedTag = False
937 938 939 940 941
		for reqSvc in self.services[0].split(' '):
			res = re.search(reqSvc, listServices)
			if res is None:
				logging.error(reqSvc + ' not found in specified docker-compose')
				self.exitStatus = 1
942 943 944
			res = re.search('oai-gnb|oai-nr-ue|oai-cu|oai-du|oai_enb|oai_ue', reqSvc)
			if res is not None:
				displayUsedTag = True
945 946 947 948
		if (self.exitStatus == 1):
			HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK)
			return

949 950 951 952 953 954 955 956 957
		cmd = 'cd ' + self.yamlPath[0] + ' && cp docker-compose.y*ml docker-compose-ci.yml'
		imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru']
		for image in imageNames:
			tagToUse = self.ImageTagToUse(image)
			cmd = f'cd {self.yamlPath[0]} && sed -i -e "s@{image}:develop@{tagToUse}@" docker-compose-ci.yml'
			subprocess.run(cmd, shell=True)
		if displayUsedTag:
			tagToUse = self.ImageTagToUse('oai-xxx')
			logging.info(f'\u001B[1m Using Image Tag: {tagToUse}\u001B[0m')
958 959

		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml up -d ' + self.services[0]
960
		logging.info(cmd)
961
		try:
962
			deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100)
963 964 965 966 967 968
		except Exception as e:
			self.exitStatus = 1
			logging.error('Could not deploy')
			HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK)
			return

969
		logging.debug('\u001B[1m Checking if all deployed healthy\u001B[0m')
970 971 972
		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps -a'
		count = 0
		healthy = 0
973
		newContainers = []
974 975
		while (count < 10):
			count += 1
976
			containerStatus = []
977
			deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
978 979
			healthy = 0
			for state in deployStatus.split('\n'):
980 981 982 983 984 985 986 987 988 989 990 991 992 993 994
				res = re.search('Name|----------', state)
				if res is not None:
					continue
				if len(state) == 0:
					continue
				res = re.search('^(?P<container_name>[a-zA-Z0-9\-\_]+) ', state)
				if res is not None:
					cName = res.group('container_name')
					found = False
					for alreadyDeployed in self.deployedContainers:
						if cName == alreadyDeployed:
							found = True
					if not found:
						newContainers.append(cName)
						self.deployedContainers.append(cName)
995
				if re.search('Up \(healthy\)', state) is not None:
996
					healthy += 1
997 998
				if re.search('rfsim4g-db-init.*Exit 0', state) is not None:
					subprocess.check_output('docker rm -f rfsim4g-db-init || true', shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
999
				containerStatus.append(state)
1000 1001 1002 1003 1004
			if healthy == self.nb_healthy[0]:
				count = 100
			else:
				time.sleep(10)

1005 1006 1007 1008 1009
		imagesInfo = ''
		for newCont in newContainers:
			cmd = 'docker inspect -f "{{.Config.Image}}" ' + newCont
			imageName = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
			imageName = str(imageName).strip()
1010
			cmd = 'docker image inspect --format "{{.RepoTags}}\t{{.Size}} bytes\t{{.Created}}\t{{.Id}}" ' + imageName
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
			imagesInfo += subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)

		html_queue = SimpleQueue()
		html_cell = '<pre style="background-color:white">\n'
		for imageInfo in imagesInfo.split('\n'):
			html_cell += imageInfo[:-11] + '\n'
		html_cell += '\n'
		for cState in containerStatus:
			html_cell += cState + '\n'
		html_cell += '</pre>'
		html_queue.put(html_cell)
1022
		if count == 100 and healthy == self.nb_healthy[0]:
1023 1024 1025
			if self.tsharkStarted == False:
				logging.debug('Starting tshark on public network')
				self.CaptureOnDockerNetworks()
1026
			HTML.CreateHtmlTestRowQueue('n/a', 'OK', 1, html_queue)
1027 1028
			for cState in containerStatus:
				logging.debug(cState)
1029 1030
			logging.info('\u001B[1m Deploying OAI Object(s) PASS\u001B[0m')
		else:
1031
			HTML.CreateHtmlTestRowQueue('Could not deploy in time', 'KO', 1, html_queue)
1032 1033
			for cState in containerStatus:
				logging.debug(cState)
1034
			logging.error('\u001B[1m Deploying OAI Object(s) FAILED\u001B[0m')
1035 1036 1037 1038 1039 1040 1041
			HTML.testCase_id = 'AUTO-UNDEPLOY'
			UE.testCase_id = 'AUTO-UNDEPLOY'
			HTML.desc = 'Automatic Undeployment'
			UE.desc = 'Automatic Undeployment'
			UE.ShowTestID()
			self.UndeployGenObject(HTML, RAN, UE)
			self.exitStatus = 1
1042

1043 1044 1045
	def CaptureOnDockerNetworks(self):
		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"'
		networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
1046
		if re.search('4g.*rfsimulator', self.yamlPath[0]) is not None:
1047 1048
			# Excluding any traffic from LTE-UE container (192.168.61.30)
			# From the trf-gen, keeping only PING traffic
1049 1050
			cmd = 'sudo nohup tshark -f "(host 192.168.61.11 and icmp) or (not host 192.168.61.11 and not host 192.168.61.30 and not arp and not port 53 and not port 2152)"'
		elif re.search('5g.*rfsimulator', self.yamlPath[0]) is not None:
1051 1052 1053
			# Excluding any traffic from NR-UE containers (192.168.71.150 and 192.168.71.151)
			# From the ext-dn, keeping only PING traffic
			cmd = 'sudo nohup tshark -f "(host 192.168.72.135 and icmp) or (not host 192.168.72.135 and not host 192.168.71.150 and not host 192.168.71.151 and not arp and not port 53 and not port 2152 and not port 2153)"'
1054 1055
		elif re.search('5g_l2sim', self.yamlPath[0]) is not None:
			cmd = 'sudo nohup tshark -f "(host 192.168.72.135 and icmp) or (not host 192.168.72.135 and not arp and not port 53 and not port 2152 and not port 2153)"'
1056 1057
		else:
			return
1058
		for name in networkNames.split('\n'):
1059
			if re.search('rfsim', name) is not None or re.search('l2sim', name) is not None:
1060 1061 1062 1063
				cmd += ' -i ' + name
		cmd += ' -w /tmp/capture_'
		ymlPath = self.yamlPath[0].split('/')
		cmd += ymlPath[1] + '.pcap > /tmp/tshark.log 2>&1 &'
1064
		logging.info(cmd)
1065
		networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
1066
		self.tsharkStarted = True
1067

1068
	def UndeployGenObject(self, HTML, RAN, UE):
1069
		self.exitStatus = 0
1070 1071
		ymlPath = self.yamlPath[0].split('/')
		logPath = '../cmake_targets/log/' + ymlPath[1]
1072

1073
		cmd = 'cd ' + self.yamlPath[0] + ' && cp docker-compose.y*ml docker-compose-ci.yml'
1074
		subprocess.run(cmd, shell=True)
1075 1076 1077 1078 1079
		imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru']
		for image in imageNames:
			tagToUse = self.ImageTagToUse(image)
			cmd = f'cd {self.yamlPath[0]} && sed -i -e "s@{image}:develop@{tagToUse}@" docker-compose-ci.yml'
			subprocess.run(cmd, shell=True)
1080

1081
		# check which containers are running for log recovery later
1082
		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps --all'
1083
		logging.info(cmd)
1084 1085 1086 1087 1088
		deployStatusLogs = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)

		# Stop the containers to shut down objects
		logging.debug('\u001B[1m Stopping containers \u001B[0m')
		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml stop'
1089
		logging.info(cmd)
1090 1091 1092 1093 1094 1095 1096 1097 1098
		try:
			deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100)
		except Exception as e:
			self.exitStatus = 1
			logging.error('Could not stop containers')
			HTML.CreateHtmlTestRow('Could not stop', 'KO', CONST.ALL_PROCESSES_OK)
			logging.error('\u001B[1m Undeploying OAI Object(s) FAILED\u001B[0m')
			return

1099
		anyLogs = False
1100
		for state in deployStatusLogs.split('\n'):
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
			res = re.search('Name|----------', state)
			if res is not None:
				continue
			if len(state) == 0:
				continue
			res = re.search('^(?P<container_name>[a-zA-Z0-9\-\_]+) ', state)
			if res is not None:
				anyLogs = True
				cName = res.group('container_name')
				cmd = 'cd ' + self.yamlPath[0] + ' && docker logs ' + cName + ' > ' + cName + '.log 2>&1'
1111
				logging.info(cmd)
1112 1113 1114
				subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
				if re.search('magma-mme', cName) is not None:
					cmd = 'cd ' + self.yamlPath[0] + ' && docker cp -L ' + cName + ':/var/log/mme.log ' + cName + '-full.log'
1115
					logging.info(cmd)
1116
					subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
1117
		fullStatus = True
1118
		if anyLogs:
1119
			cmd = 'mkdir -p '+ logPath + ' && cp ' + self.yamlPath[0] + '/*.log ' + logPath
1120
			logging.info(cmd)
1121 1122
			deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)

1123
			# Analyzing log file(s)!
1124 1125
			listOfPossibleRanContainers = ['enb', 'gnb', 'cu', 'du']
			for container in listOfPossibleRanContainers:
1126
				filenames = self.yamlPath[0] + '/*-oai-' + container + '.log'
1127
				cmd = 'ls ' + filenames
1128 1129 1130
				containerStatus = True
				try:
					lsStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
1131
					filenames = str(lsStatus).strip()
1132 1133 1134 1135 1136
				except:
					containerStatus = False
				if not containerStatus:
					continue

1137 1138 1139 1140 1141 1142 1143 1144 1145
				for filename in filenames.split('\n'):
					logging.debug('\u001B[1m Analyzing xNB logfile ' + filename + ' \u001B[0m')
					logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers)
					if (logStatus < 0):
						fullStatus = False
						self.exitStatus = 1
						HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
					else:
						HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
1146

1147 1148
			listOfPossibleUeContainers = ['lte-ue*', 'nr-ue*']
			for container in listOfPossibleUeContainers:
1149
				filenames = self.yamlPath[0] + '/*-oai-' + container + '.log'
1150
				cmd = 'ls ' + filenames
1151 1152 1153
				containerStatus = True
				try:
					lsStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
1154
					filenames = str(lsStatus).strip()
1155 1156 1157 1158 1159
				except:
					containerStatus = False
				if not containerStatus:
					continue

1160 1161 1162 1163 1164 1165 1166 1167
				for filename in filenames.split('\n'):
					logging.debug('\u001B[1m Analyzing UE logfile ' + filename + ' \u001B[0m')
					logStatus = UE.AnalyzeLogFile_UE(filename, HTML, RAN)
					if (logStatus < 0):
						fullStatus = False
						HTML.CreateHtmlTestRow('UE log Analysis', 'KO', logStatus)
					else:
						HTML.CreateHtmlTestRow('UE log Analysis', 'OK', CONST.ALL_PROCESSES_OK)
1168

1169
			cmd = 'rm ' + self.yamlPath[0] + '/*.log'
1170
			logging.info(cmd)
1171
			deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
1172 1173 1174 1175
			if self.tsharkStarted:
				self.tsharkStarted = True
				ymlPath = self.yamlPath[0].split('/')
				cmd = 'sudo chmod 666 /tmp/capture_' + ymlPath[1] + '.pcap'
1176
				logging.info(cmd)
1177 1178
				copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
				cmd = 'cp /tmp/capture_' + ymlPath[1] + '.pcap ' + logPath
1179
				logging.info(cmd)
1180 1181
				copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
				cmd = 'sudo rm /tmp/capture_' + ymlPath[1] + '.pcap'
1182
				logging.info(cmd)
1183
				copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
1184
				self.tsharkStarted = False
1185

1186
		logging.debug('\u001B[1m Undeploying \u001B[0m')
1187
		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml down'
1188
		logging.info(cmd)
1189 1190 1191 1192 1193 1194 1195 1196 1197
		try:
			deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100)
		except Exception as e:
			self.exitStatus = 1
			logging.error('Could not undeploy')
			HTML.CreateHtmlTestRow('Could not undeploy', 'KO', CONST.ALL_PROCESSES_OK)
			logging.error('\u001B[1m Undeploying OAI Object(s) FAILED\u001B[0m')
			return

1198
		self.deployedContainers = []
1199 1200
		# Cleaning any created tmp volume
		cmd = 'docker volume prune --force || true'
1201
		logging.info(cmd)
1202 1203
		deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100)

1204 1205 1206 1207 1208
		if fullStatus:
			HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK)
			logging.info('\u001B[1m Undeploying OAI Object(s) PASS\u001B[0m')
		else:
			HTML.CreateHtmlTestRow('n/a', 'KO', CONST.ALL_PROCESSES_OK)
1209
			logging.error('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m')
1210

1211 1212 1213 1214 1215 1216 1217
	def StatsFromGenObject(self, HTML):
		self.exitStatus = 0
		ymlPath = self.yamlPath[0].split('/')
		logPath = '../cmake_targets/log/' + ymlPath[1]

		# if the containers are running, recover the logs!
		cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps --all'
1218
		logging.info(cmd)
1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
		deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
		cmd = 'docker stats --no-stream --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}" '
		anyLogs = False
		for state in deployStatus.split('\n'):
			res = re.search('Name|----------', state)
			if res is not None:
				continue
			if len(state) == 0:
				continue
			res = re.search('^(?P<container_name>[a-zA-Z0-9\-\_]+) ', state)
			if res is not None:
				anyLogs = True
				cmd += res.group('container_name') + ' '
		message = ''
		if anyLogs:
1234
			logging.info(cmd)
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
			stats = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
			for statLine in stats.split('\n'):
				logging.debug(statLine)
				message += statLine + '\n'

		html_queue = SimpleQueue()
		html_cell = '<pre style="background-color:white">\n' + message + '</pre>'
		html_queue.put(html_cell)
		HTML.CreateHtmlTestRowQueue(self.pingOptions, 'OK', 1, html_queue)

1245
	def PingFromContainer(self, HTML, RAN, UE):
1246
		self.exitStatus = 0
1247 1248 1249
		ymlPath = self.yamlPath[0].split('/')
		logPath = '../cmake_targets/log/' + ymlPath[1]
		cmd = 'mkdir -p ' + logPath
1250 1251
		deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)

1252
		cmd = 'docker exec ' + self.pingContName + ' /bin/bash -c "ping ' + self.pingOptions + '" 2>&1 | tee ' + logPath + '/ping_' + HTML.testCase_id + '.log || true'
1253

1254
		logging.info(cmd)
1255 1256 1257 1258
		deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100)

		result = re.search(', (?P<packetloss>[0-9\.]+)% packet loss, time [0-9\.]+ms', deployStatus)
		if result is None:
1259
			self.PingExit(HTML, RAN, UE, False, 'Packet Loss Not Found')
1260 1261 1262 1263
			return

		packetloss = result.group('packetloss')
		if float(packetloss) == 100:
1264
			self.PingExit(HTML, RAN, UE, False, 'Packet Loss is 100%')
1265 1266 1267 1268
			return

		result = re.search('rtt min\/avg\/max\/mdev = (?P<rtt_min>[0-9\.]+)\/(?P<rtt_avg>[0-9\.]+)\/(?P<rtt_max>[0-9\.]+)\/[0-9\.]+ ms', deployStatus)
		if result is None:
1269
			self.PingExit(HTML, RAN, UE, False, 'Ping RTT_Min RTT_Avg RTT_Max Not Found!')
1270
			return
1271

1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
		rtt_min = result.group('rtt_min')
		rtt_avg = result.group('rtt_avg')
		rtt_max = result.group('rtt_max')
		pal_msg = 'Packet Loss : ' + packetloss + '%'
		min_msg = 'RTT(Min)    : ' + rtt_min + ' ms'
		avg_msg = 'RTT(Avg)    : ' + rtt_avg + ' ms'
		max_msg = 'RTT(Max)    : ' + rtt_max + ' ms'

		message = 'ping result\n'
		message += '    ' + pal_msg + '\n'
		message += '    ' + min_msg + '\n'
		message += '    ' + avg_msg + '\n'
		message += '    ' + max_msg + '\n'
		packetLossOK = True
		if float(packetloss) > float(self.pingLossThreshold):
			message += '\nPacket Loss too high'
			packetLossOK = False
		elif float(packetloss) > 0:
			message += '\nPacket Loss is not 0%'
1291
		self.PingExit(HTML, RAN, UE, packetLossOK, message)
1292 1293 1294 1295 1296 1297 1298 1299 1300

		if packetLossOK:
			logging.debug('\u001B[1;37;44m ping result \u001B[0m')
			logging.debug('\u001B[1;34m    ' + pal_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + min_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + avg_msg + '\u001B[0m')
			logging.debug('\u001B[1;34m    ' + max_msg + '\u001B[0m')
			logging.info('\u001B[1m Ping Test PASS\u001B[0m')

1301
	def PingExit(self, HTML, RAN, UE, status, message):
1302 1303 1304 1305 1306 1307
		html_queue = SimpleQueue()
		html_cell = '<pre style="background-color:white">UE\n' + message + '</pre>'
		html_queue.put(html_cell)
		if status:
			HTML.CreateHtmlTestRowQueue(self.pingOptions, 'OK', 1, html_queue)
		else:
1308
			logging.error('\u001B[1;37;41m ping test FAIL -- ' + message + ' \u001B[0m')
1309
			HTML.CreateHtmlTestRowQueue(self.pingOptions, 'KO', 1, html_queue)
1310
			# Automatic undeployment
1311 1312 1313
			logging.warning('----------------------------------------')
			logging.warning('\u001B[1m Starting Automatic undeployment \u001B[0m')
			logging.warning('----------------------------------------')
1314 1315
			HTML.testCase_id = 'AUTO-UNDEPLOY'
			HTML.desc = 'Automatic Un-Deployment'
1316
			self.UndeployGenObject(HTML, RAN, UE)
1317
			self.exitStatus = 1
1318

1319
	def IperfFromContainer(self, HTML, RAN):
1320 1321
		self.exitStatus = 0

1322 1323 1324
		ymlPath = self.yamlPath[0].split('/')
		logPath = '../cmake_targets/log/' + ymlPath[1]
		cmd = 'mkdir -p ' + logPath
1325 1326 1327
		logStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)

		# Start the server process
1328
		cmd = 'docker exec -d ' + self.svrContName + ' /bin/bash -c "nohup iperf ' + self.svrOptions + ' > /tmp/iperf_server.log 2>&1" || true'
1329
		logging.info(cmd)
1330 1331 1332 1333
		serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
		time.sleep(5)

		# Start the client process
1334

1335
		cmd = 'docker exec ' + self.cliContName + ' /bin/bash -c "iperf ' + self.cliOptions + '" 2>&1 | tee '+ logPath + '/iperf_client_' + HTML.testCase_id + '.log || true'
1336
		logging.info(cmd)
1337 1338 1339
		clientStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100)

		# Stop the server process
1340
		cmd = 'docker exec ' + self.svrContName + ' /bin/bash -c "pkill iperf" || true'
1341
		logging.info(cmd)
1342 1343
		serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
		time.sleep(5)
1344
		cmd = 'docker cp ' + self.svrContName + ':/tmp/iperf_server.log '+ logPath + '/iperf_server_' + HTML.testCase_id + '.log'
1345
		logging.info(cmd)
1346
		serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
1347 1348 1349 1350 1351 1352 1353 1354 1355 1356

		# Analyze client output
		result = re.search('Server Report:', clientStatus)
		if result is None:
			result = re.search('read failed: Connection refused', clientStatus)
			if result is not None:
				message = 'Could not connect to iperf server!'
			else:
				message = 'Server Report and Connection refused Not Found!'
			self.IperfExit(HTML, False, message)
1357
			logging.error('\u001B[1;37;41m Iperf Test FAIL\u001B[0m')
1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424
			return

		# Computing the requested bandwidth in float
		result = re.search('-b (?P<iperf_bandwidth>[0-9\.]+)[KMG]', self.cliOptions)
		if result is not None:
			req_bandwidth = result.group('iperf_bandwidth')
			req_bw = float(req_bandwidth)
			result = re.search('-b [0-9\.]+K', self.cliOptions)
			if result is not None:
				req_bandwidth = '%.1f Kbits/sec' % req_bw
				req_bw = req_bw * 1000
			result = re.search('-b [0-9\.]+M', self.cliOptions)
			if result is not None:
				req_bandwidth = '%.1f Mbits/sec' % req_bw
				req_bw = req_bw * 1000000

		reportLine = None
		reportLineFound = False
		for iLine in clientStatus.split('\n'):
			if reportLineFound:
				reportLine = iLine
				reportLineFound = False
			res = re.search('Server Report:', iLine)
			if res is not None:
				reportLineFound = True
		result = None
		if reportLine is not None:
			result = re.search('(?:|\[ *\d+\].*) (?P<bitrate>[0-9\.]+ [KMG]bits\/sec) +(?P<jitter>[0-9\.]+ ms) +(\d+\/ ..\d+) +(\((?P<packetloss>[0-9\.]+)%\))', reportLine)
		iperfStatus = True
		if result is not None:
			bitrate = result.group('bitrate')
			packetloss = result.group('packetloss')
			jitter = result.group('jitter')
			logging.debug('\u001B[1;37;44m iperf result \u001B[0m')
			iperfStatus = True
			msg = 'Req Bitrate : ' + req_bandwidth + '\n'
			logging.debug('\u001B[1;34m    Req Bitrate : ' + req_bandwidth + '\u001B[0m')
			if bitrate is not None:
				msg += 'Bitrate     : ' + bitrate + '\n'
				logging.debug('\u001B[1;34m    Bitrate     : ' + bitrate + '\u001B[0m')
				result = re.search('(?P<real_bw>[0-9\.]+) [KMG]bits/sec', str(bitrate))
				if result is not None:
					actual_bw = float(str(result.group('real_bw')))
					result = re.search('[0-9\.]+ K', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000
					result = re.search('[0-9\.]+ M', bitrate)
					if result is not None:
						actual_bw = actual_bw * 1000000
					br_loss = 100 * actual_bw / req_bw
					if br_loss < 90:
						iperfStatus = False
					bitperf = '%.2f ' % br_loss
					msg += 'Bitrate Perf: ' + bitperf + '%\n'
					logging.debug('\u001B[1;34m    Bitrate Perf: ' + bitperf + '%\u001B[0m')
			if packetloss is not None:
				msg += 'Packet Loss : ' + packetloss + '%\n'
				logging.debug('\u001B[1;34m    Packet Loss : ' + packetloss + '%\u001B[0m')
				if float(packetloss) > float(5):
					msg += 'Packet Loss too high!\n'
					logging.debug('\u001B[1;37;41m Packet Loss too high \u001B[0m')
					iperfStatus = False
			if jitter is not None:
				msg += 'Jitter      : ' + jitter + '\n'
				logging.debug('\u001B[1;34m    Jitter      : ' + jitter + '\u001B[0m')
			self.IperfExit(HTML, iperfStatus, msg)
		else:
1425
			iperfStatus = False
1426
			logging.error('problem?')
1427
			self.IperfExit(HTML, iperfStatus, 'problem?')
1428 1429
		if iperfStatus:
			logging.info('\u001B[1m Iperf Test PASS\u001B[0m')
1430 1431
		else:
			logging.error('\u001B[1;37;41m Iperf Test FAIL\u001B[0m')
1432 1433 1434 1435 1436 1437 1438 1439

	def IperfExit(self, HTML, status, message):
		html_queue = SimpleQueue()
		html_cell = '<pre style="background-color:white">UE\n' + message + '</pre>'
		html_queue.put(html_cell)
		if status:
			HTML.CreateHtmlTestRowQueue(self.cliOptions, 'OK', 1, html_queue)
		else:
1440
			logging.error('\u001B[1m Iperf Test FAIL -- ' + message + ' \u001B[0m')
1441
			HTML.CreateHtmlTestRowQueue(self.cliOptions, 'KO', 1, html_queue)
1442

Laurent THOMAS's avatar
Laurent THOMAS committed
1443

1444 1445 1446 1447 1448 1449
	def CheckAndAddRoute(self, svrName, ipAddr, userName, password):
		logging.debug('Checking IP routing on ' + svrName)
		mySSH = SSH.SSHConnection()
		if svrName == 'porcepix':
			mySSH.open(ipAddr, userName, password)
			# Check if route to asterix gnb exists
1450
			mySSH.command('ip route | grep --colour=never "192.168.68.64/26"', '\$', 10)
1451
			result = re.search('172.21.16.127', mySSH.getBefore())
1452
			if result is None:
1453
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.64/26 via 172.21.16.127 dev eno1', '\$', 10)
1454
			# Check if route to obelix enb exists
1455
			mySSH.command('ip route | grep --colour=never "192.168.68.128/26"', '\$', 10)
1456
			result = re.search('172.21.16.128', mySSH.getBefore())
1457
			if result is None:
1458
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.128/26 via 172.21.16.128 dev eno1', '\$', 10)
1459
			# Check if route to nepes gnb exists
1460
			mySSH.command('ip route | grep --colour=never "192.168.68.192/26"', '\$', 10)
1461
			result = re.search('172.21.16.137', mySSH.getBefore())
1462
			if result is None:
1463
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.192/26 via 172.21.16.137 dev eno1', '\$', 10)
1464 1465 1466 1467 1468 1469
			# Check if forwarding is enabled
			mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
			result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
			if result is None:
				mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
			# Check if iptables forwarding is accepted
1470
			mySSH.command('echo ' + password + ' | sudo -S iptables -L FORWARD', '\$', 10)
1471 1472 1473 1474 1475 1476 1477
			result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
			if result is None:
				mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
			mySSH.close()
		if svrName == 'asterix':
			mySSH.open(ipAddr, userName, password)
			# Check if route to porcepix epc exists
1478
			mySSH.command('ip route | grep --colour=never "192.168.61.192/26"', '\$', 10)
1479
			result = re.search('172.21.16.136', mySSH.getBefore())
1480
			if result is None:
1481
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.61.192/26 via 172.21.16.136 dev em1', '\$', 10)
1482
			# Check if route to porcepix cn5g exists
1483
			mySSH.command('ip route | grep --colour=never "192.168.70.128/26"', '\$', 10)
1484
			result = re.search('172.21.16.136', mySSH.getBefore())
1485
			if result is None:
1486
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.70.128/26 via 172.21.16.136 dev em1', '\$', 10)
1487
			# Check if X2 route to obelix enb exists
1488
			mySSH.command('ip route | grep --colour=never "192.168.68.128/26"', '\$', 10)
1489
			result = re.search('172.21.16.128', mySSH.getBefore())
1490
			if result is None:
1491
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.128/26 via 172.21.16.128 dev em1', '\$', 10)
1492 1493 1494 1495 1496 1497
			# Check if forwarding is enabled
			mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
			result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
			if result is None:
				mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
			# Check if iptables forwarding is accepted
1498
			mySSH.command('echo ' + password + ' | sudo -S iptables -L FORWARD', '\$', 10)
1499 1500 1501 1502 1503 1504 1505
			result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
			if result is None:
				mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
			mySSH.close()
		if svrName == 'obelix':
			mySSH.open(ipAddr, userName, password)
			# Check if route to porcepix epc exists
1506
			mySSH.command('ip route | grep --colour=never "192.168.61.192/26"', '\$', 10)
1507
			result = re.search('172.21.16.136', mySSH.getBefore())
1508
			if result is None:
1509
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.61.192/26 via 172.21.16.136 dev eno1', '\$', 10)
1510
			# Check if X2 route to asterix gnb exists
1511
			mySSH.command('ip route | grep --colour=never "192.168.68.64/26"', '\$', 10)
1512
			result = re.search('172.21.16.127', mySSH.getBefore())
1513
			if result is None:
1514
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.64/26 via 172.21.16.127 dev eno1', '\$', 10)
1515
			# Check if X2 route to nepes gnb exists
1516
			mySSH.command('ip route | grep --colour=never "192.168.68.192/26"', '\$', 10)
1517
			result = re.search('172.21.16.137', mySSH.getBefore())
1518
			if result is None:
1519
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.192/26 via 172.21.16.137 dev eno1', '\$', 10)
1520 1521 1522 1523 1524 1525
			# Check if forwarding is enabled
			mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
			result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
			if result is None:
				mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
			# Check if iptables forwarding is accepted
1526
			mySSH.command('echo ' + password + ' | sudo -S iptables -L FORWARD', '\$', 10)
1527 1528 1529 1530 1531 1532 1533
			result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
			if result is None:
				mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
			mySSH.close()
		if svrName == 'nepes':
			mySSH.open(ipAddr, userName, password)
			# Check if route to porcepix epc exists
1534
			mySSH.command('ip route | grep --colour=never "192.168.61.192/26"', '\$', 10)
1535
			result = re.search('172.21.16.136', mySSH.getBefore())
1536
			if result is None:
1537
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.61.192/26 via 172.21.16.136 dev enp0s31f6', '\$', 10)
1538
			# Check if X2 route to obelix enb exists
1539
			mySSH.command('ip route | grep --colour=never "192.168.68.128/26"', '\$', 10)
1540
			result = re.search('172.21.16.128', mySSH.getBefore())
1541
			if result is None:
1542
				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.128/26 via 172.21.16.128 dev enp0s31f6', '\$', 10)
1543 1544 1545 1546 1547 1548
			# Check if forwarding is enabled
			mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
			result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
			if result is None:
				mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
			# Check if iptables forwarding is accepted
1549
			mySSH.command('echo ' + password + ' | sudo -S iptables -L FORWARD', '\$', 10)
1550 1551 1552 1553
			result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
			if result is None:
				mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
			mySSH.close()
Laurent THOMAS's avatar
Laurent THOMAS committed
1554