cls_containerize.py 38.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
34 35
import sys	      # arg
import re	       # reg
36 37
import logging
import os
38
import shutil
39
import time
40
from zipfile import ZipFile
41 42 43 44

#-----------------------------------------------------------
# OAI Testing modules
#-----------------------------------------------------------
45
import cls_cmd
46 47
import helpreadme as HELP
import constants as CONST
48
import cls_oaicitest
49

50 51 52 53
#-----------------------------------------------------------
# Helper functions used here and in other classes
# (e.g., cls_cluster.py)
#-----------------------------------------------------------
54
IMAGES = ['oai-enb', 'oai-lte-ru', 'oai-lte-ue', 'oai-gnb', 'oai-nr-cuup', 'oai-gnb-aw2s', 'oai-nr-ue', 'oai-enb-asan', 'oai-gnb-asan', 'oai-lte-ue-asan', 'oai-nr-ue-asan', 'oai-nr-cuup-asan', 'oai-gnb-aerial', 'oai-gnb-fhi72']
55

56
def CreateWorkspace(host, sourcePath, ranRepository, ranCommitID, ranTargetBranch, ranAllowMerge):
57 58
	if ranCommitID == '':
		logging.error('need ranCommitID in CreateWorkspace()')
59 60 61 62
		raise ValueError('Insufficient Parameter in CreateWorkspace(): need ranCommitID')

	script = "scripts/create_workspace.sh"
	options = f"{sourcePath} {ranRepository} {ranCommitID}"
63 64
	if ranAllowMerge:
		if ranTargetBranch == '':
65
			ranTargetBranch = 'develop'
66 67 68 69 70
		options += f" {ranTargetBranch}"
	logging.info(f'execute "{script}" with options "{options}" on node {host}')
	ret = cls_cmd.runScript(host, script, 90, options)
	logging.debug(f'"{script}" finished with code {ret.returncode}, output:\n{ret.stdout}')
	return ret.returncode == 0
71

72
def CreateTag(ranCommitID, ranBranch, ranAllowMerge):
73 74
	if ranCommitID == 'develop':
		return 'develop'
75 76
	shortCommit = ranCommitID[0:8]
	if ranAllowMerge:
77 78 79
		# Allowing contributor to have a name/branchName format
		branchName = ranBranch.replace('/','-')
		tagToUse = f'{branchName}-{shortCommit}'
80 81
	else:
		tagToUse = f'develop-{shortCommit}'
82
	return tagToUse
83

84 85 86 87 88 89
def CopyLogsToExecutor(cmd, sourcePath, log_name):
	cmd.cd(f'{sourcePath}/cmake_targets')
	cmd.run(f'rm -f {log_name}.zip')
	cmd.run(f'mkdir -p {log_name}')
	cmd.run(f'mv log/* {log_name}')
	cmd.run(f'zip -r -qq {log_name}.zip {log_name}')
90 91 92 93 94 95

	# copy zip to executor for analysis
	if (os.path.isfile(f'./{log_name}.zip')):
		os.remove(f'./{log_name}.zip')
	if (os.path.isdir(f'./{log_name}')):
		shutil.rmtree(f'./{log_name}')
96 97
	cmd.copyin(f'{sourcePath}/cmake_targets/{log_name}.zip', f'./{log_name}.zip')
	cmd.run(f'rm -f {log_name}.zip')
98 99
	ZipFile(f'{log_name}.zip').extractall('.')

100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
def AnalyzeBuildLogs(buildRoot, images, globalStatus):
	collectInfo = {}
	for image in images:
		files = {}
		file_list = [f for f in os.listdir(f'{buildRoot}/{image}') if os.path.isfile(os.path.join(f'{buildRoot}/{image}', f)) and f.endswith('.txt')]
		# Analyze the "sub-logs" of every target image
		for fil in file_list:
			errorandwarnings = {}
			warningsNo = 0
			errorsNo = 0
			with open(f'{buildRoot}/{image}/{fil}', mode='r') as inputfile:
				for line in inputfile:
					result = re.search(' ERROR ', str(line))
					if result is not None:
						errorsNo += 1
					result = re.search(' error:', str(line))
					if result is not None:
						errorsNo += 1
					result = re.search(' WARNING ', str(line))
					if result is not None:
						warningsNo += 1
					result = re.search(' warning:', str(line))
					if result is not None:
						warningsNo += 1
				errorandwarnings['errors'] = errorsNo
				errorandwarnings['warnings'] = warningsNo
				errorandwarnings['status'] = globalStatus
			files[fil] = errorandwarnings
		# Analyze the target image
		if os.path.isfile(f'{buildRoot}/{image}.log'):
			errorandwarnings = {}
			committed = False
			tagged = False
			with open(f'{buildRoot}/{image}.log', mode='r') as inputfile:
				for line in inputfile:
135 136 137 138 139 140
					lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None
					lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None
					tagged = tagged or lineHasTag or lineHasTag2
					# the OpenShift Cluster builder prepends image registry URL
					lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None
					committed = committed or lineHasCommit
141 142 143 144 145 146 147
			errorandwarnings['errors'] = 0 if committed or tagged else 1
			errorandwarnings['warnings'] = 0
			errorandwarnings['status'] = committed or tagged
			files['Target Image Creation'] = errorandwarnings
		collectInfo[image] = files
	return collectInfo

148 149 150 151
def GetImageName(ssh, svcName, file):
	ret = ssh.run(f"docker compose -f {file} config --format json {svcName}  | jq -r '.services.\"{svcName}\".image'", silent=True)
	if ret.returncode != 0:
		return f"cannot retrieve image info for {containerName}: {ret.stdout}"
152
	else:
153 154
		return ret.stdout.strip()

155 156 157 158 159 160
def GetServiceHealth(ssh, svcName, file):
	if svcName is None:
		return False, f"Service {svcName} not found in {file}"
	image = GetImageName(ssh, svcName, file)
	if 'db_init' in svcName or 'db-init' in svcName: # exits with 0, there cannot be healthy
		return True, f"Service {svcName} healthy, image {image}"
161
	for _ in range(8):
162
		result = ssh.run(f"docker compose -f {file} ps --format json {svcName}  | jq -r 'if type==\"array\" then .[0].Health else .Health end'", silent=True)
163
		if result.stdout == 'healthy':
164
			return True, f"Service {svcName} healthy, image {image}"
165
		time.sleep(5)
166
	return False, f"Failed to deploy: service {svcName}"
167 168

def ExistEnvFilePrint(ssh, wd, prompt='env vars in existing'):
169
	ret = ssh.run(f'cat {wd}/.env', silent=True, reportNonZero=False)
170 171 172 173 174 175
	if ret.returncode != 0:
		return False
	env_vars = ret.stdout.strip().splitlines()
	logging.info(f'{prompt} {wd}/.env: {env_vars}')
	return True

176
def WriteEnvFile(ssh, services, wd, tag, flexric_tag):
177
	ret = ssh.run(f'cat {wd}/.env', silent=True, reportNonZero=False)
178
	registry = "oai-ci" # pull_images() gives us this registry path
179
	envs = {"REGISTRY":registry, "TAG": tag, "FLEXRIC_TAG": flexric_tag}
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
	if ret.returncode == 0: # it exists, we have to update
		# transforms env file to dictionary
		old_envs = {}
		for l in ret.stdout.strip().splitlines():
			var, val = l.split('=', 1)
			old_envs[var] = val.strip('"')
		# will retain the old environment variables
		envs = {**envs, **old_envs}
	for svc in services.split():
		# In some scenarios we have the choice of either pulling normal images
		# or -asan images. We need to detect which kind we did pull.
		fullImageName = GetImageName(ssh, svc, f"{wd}/docker-compose.y*ml")
		image = fullImageName.split("/")[-1].split(":")[0]
		checkimg = f"{registry}/{image}-asan:{tag}"
		ret = ssh.run(f'docker image inspect {checkimg}', reportNonZero=False)
		if ret.returncode == 0:
			logging.info(f"detected pulled image {checkimg}")
197 198 199
			if "oai-enb" in image: envs["ENB_IMG"] = "oai-enb-asan"
			elif "oai-gnb" in image: envs["GNB_IMG"] = "oai-gnb-asan"
			elif "oai-lte-ue" in image: envs["LTEUE_IMG"] = "oai-lte-ue-asan"
200 201 202 203 204 205 206 207 208 209 210 211 212
			elif "oai-nr-ue" in image: envs["NRUE_IMG"] = "oai-nr-ue-asan"
			elif "oai-nr-cuup" in image: envs["NRCUUP_IMG"] = "oai-nr-cuup-asan"
			else: logging.warning("undetected image format {image}, cannot use asan")
	env_string = "\n".join([f"{var}=\"{val}\"" for var,val in envs.items()])
	ssh.run(f'echo -e \'{env_string}\' > {wd}/.env', silent=True)
	ExistEnvFilePrint(ssh, wd, prompt='New env vars in file')

def GetServices(ssh, requested, file):
    if requested == [] or requested is None or requested == "":
        logging.warning('No service name given: starting all services in docker-compose.yml!')
        ret = ssh.run(f'docker compose -f {file} config --services')
        if ret.returncode != 0:
            return ""
213
        else:
214 215 216 217
            return ' '.join(ret.stdout.splitlines())
    else:
        return requested

218
def CopyinServiceLog(ssh, lSourcePath, yaml, svcName, wd_yaml, filename):
219
	remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}"
220
	ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}')
221 222 223 224 225 226 227 228
	local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}"
	local_filename = f"{local_dir}/{filename}"
	return ssh.copyin(remote_filename, local_filename)

def GetRunningServices(ssh, file):
	ret = ssh.run(f'docker compose -f {file} config --services')
	if ret.returncode != 0:
		return None
229
	allServices = ret.stdout.splitlines()
230
	running_services = []
231 232
	for s in allServices:
		# outputs the hash if the container is running
233 234 235 236 237 238 239 240 241 242 243 244 245 246
		ret = ssh.run(f'docker compose -f {file} ps --all --quiet -- {s}')
		if ret.returncode != 0:
			logging.info(f"service {s}: {ret.stdout}")
		elif ret.stdout == "":
			logging.warning(f"could not retrieve information for service {s}")
		else:
			c = ret.stdout
			logging.debug(f'running service {s} with container id {c}')
			running_services.append((s, c))
	logging.info(f'stopping services: {running_services}')
	return running_services

def CheckLogs(self, yaml, service_name, HTML, RAN):
	logPath = f'{os.getcwd()}/../cmake_targets/log/{yaml}'
247
	filename = f'{logPath}/{service_name}-{HTML.testCase_id}.log'
248
	success = True
249 250 251
	if (any(sub in service_name for sub in ['oai_ue','oai-nr-ue','lte_ue'])):
		logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m')
		logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(filename, HTML, RAN)
252 253 254 255 256
		opt = f"UE log analysis for service {service_name}"
		# usage of htmlUEFailureMsg/htmleNBFailureMsg is because Analyze log files
		# abuse HTML to store their reports, and we here want to put custom options,
		# which is not possible with CreateHtmlTestRow
		# solution: use HTML templates, where we don't need different HTML write funcs
257
		if (logStatus < 0):
258 259
			HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmlUEFailureMsg])
			success = False
260
		else:
261 262
			HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmlUEFailureMsg])
		HTML.htmlUEFailureMsg = ""
263 264 265 266 267 268
	elif service_name == 'nv-cubb':
		msg = 'Undeploy PNF/Nvidia CUBB'
		HTML.CreateHtmlTestRow(msg, 'OK', CONST.ALL_PROCESSES_OK)
	elif (any(sub in service_name for sub in ['enb','rru','rcc','cu','du','gnb'])):
		logging.debug(f'\u001B[1m Analyzing XnB logfile {filename}\u001B[0m')
		logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers)
269
		opt = f"xNB log analysis for service {service_name}"
270
		if (logStatus < 0):
271 272
			HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmleNBFailureMsg])
			success = False
273
		else:
274
			HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg])
275
		HTML.htmleNBFailureMsg = ""
276 277
	else:
		logging.info(f'Skipping to analyze log for service name {service_name}')
278 279 280
		HTML.CreateHtmlTestRowQueue(f"service {service_name}", 'OK', ["no analysis function"])
	logging.debug(f"log check: service {service_name} passed analysis {success}")
	return success
281

282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class Containerize():

	def __init__(self):
		
		self.ranRepository = ''
		self.ranBranch = ''
		self.ranAllowMerge = False
		self.ranCommitID = ''
		self.ranTargetBranch = ''
		self.eNBIPAddress = ''
		self.eNBUserName = ''
		self.eNBPassword = ''
		self.eNBSourceCodePath = ''
		self.eNB1IPAddress = ''
		self.eNB1UserName = ''
		self.eNB1Password = ''
		self.eNB1SourceCodePath = ''
		self.eNB2IPAddress = ''
		self.eNB2UserName = ''
		self.eNB2Password = ''
		self.eNB2SourceCodePath = ''
		self.forcedWorkspaceCleanup = False
		self.imageKind = ''
308
		self.proxyCommit = None
309 310
		self.eNB_instance = 0
		self.eNB_serverId = ['', '', '']
311
		self.yamlPath = ['', '', '']
312
		self.services = ['', '', '']
313
		self.deploymentTag = ''
314 315
		self.eNB_logFile = ['', '', '']

316
		self.testCase_id = ''
317

318
		self.cli = ''
319
		self.cliBuildOptions = ''
320 321
		self.dockerfileprefix = ''
		self.host = ''
322

323
		self.deployedContainers = []
324
		self.tsharkStarted = False
325 326 327 328 329 330 331 332
		self.pingContName = ''
		self.pingOptions = ''
		self.pingLossThreshold = ''
		self.svrContName = ''
		self.svrOptions = ''
		self.cliContName = ''
		self.cliOptions = ''

333
		self.imageToCopy = ''
334 335
		#checkers from xml
		self.ran_checkers={}
336
		self.num_attempts = 1
337

338 339
		self.flexricTag = ''

340 341 342 343
#-----------------------------------------------------------
# Container management functions
#-----------------------------------------------------------

344 345 346 347 348 349 350 351 352 353
	def GetCredentials(self, server_id):
		if server_id == '0':
			ip, path = self.eNBIPAddress, self.eNBSourceCodePath
		elif server_id == '1':
			ip, path = self.eNB1IPAddress, self.eNB1SourceCodePath
		elif server_id == '2':
			ip, path = self.eNB2IPAddress, self.eNB2SourceCodePath
		else:
			raise ValueError(f"unknown server ID '{server_id}'")
		if ip == '' or path == '':
354
			HELP.GenericHelp(CONST.Version)
355 356 357 358 359 360
			raise ValueError(f'Insufficient Parameter: IP/node {ip}, path {path}')
		return (ip, path)

	def BuildImage(self, HTML):
		svr = self.eNB_serverId[self.eNB_instance]
		lIpAddr, lSourcePath = self.GetCredentials(svr)
361
		logging.debug('Building on server: ' + lIpAddr)
362
		cmd = cls_cmd.RemoteCmd(lIpAddr)
363 364
	
		# Checking the hostname to get adapted on cli and dockerfileprefixes
365 366
		cmd.run('hostnamectl')
		result = re.search('Ubuntu|Red Hat', cmd.getBefore())
367 368 369
		self.host = result.group(0)
		if self.host == 'Ubuntu':
			self.cli = 'docker'
370
			self.dockerfileprefix = '.ubuntu22'
371
			self.cliBuildOptions = ''
372
		elif self.host == 'Red Hat':
373
			self.cli = 'sudo podman'
374
			self.dockerfileprefix = '.rhel9'
375
			self.cliBuildOptions = '--disable-compression'
376

377
		# we always build the ran-build image with all targets
378 379
		# Creating a tupple with the imageName, the DockerFile prefix pattern, targetName and sanitized option
		imageNames = [('ran-build', 'build', 'ran-build', '')]
380 381
		result = re.search('eNB', self.imageKind)
		if result is not None:
382
			imageNames.append(('oai-enb', 'eNB', 'oai-enb', ''))
383 384
		result = re.search('gNB', self.imageKind)
		if result is not None:
385
			imageNames.append(('oai-gnb', 'gNB', 'oai-gnb', ''))
386 387
		result = re.search('all', self.imageKind)
		if result is not None:
388 389 390 391 392
			imageNames.append(('oai-enb', 'eNB', 'oai-enb', ''))
			imageNames.append(('oai-gnb', 'gNB', 'oai-gnb', ''))
			imageNames.append(('oai-nr-cuup', 'nr-cuup', 'oai-nr-cuup', ''))
			imageNames.append(('oai-lte-ue', 'lteUE', 'oai-lte-ue', ''))
			imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', ''))
393
			if self.host == 'Red Hat':
394
				imageNames.append(('oai-physim', 'phySim', 'oai-physim', ''))
395
			if self.host == 'Ubuntu':
396
				imageNames.append(('oai-lte-ru', 'lteRU', 'oai-lte-ru', ''))
Robert Schmidt's avatar
Robert Schmidt committed
397
				imageNames.append(('oai-gnb-aerial', 'gNB.aerial', 'oai-gnb-aerial', ''))
398
				# Building again the 5G images with Address Sanitizer
399
				imageNames.append(('ran-build', 'build', 'ran-build-asan', '--build-arg "BUILD_OPTION=--sanitize"'))
400
				imageNames.append(('oai-enb', 'eNB', 'oai-enb-asan', '--build-arg "BUILD_OPTION=--sanitize"'))
401
				imageNames.append(('oai-gnb', 'gNB', 'oai-gnb-asan', '--build-arg "BUILD_OPTION=--sanitize"'))
402
				imageNames.append(('oai-lte-ue', 'lteUE', 'oai-lte-ue-asan', '--build-arg "BUILD_OPTION=--sanitize"'))
403 404
				imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue-asan', '--build-arg "BUILD_OPTION=--sanitize"'))
				imageNames.append(('oai-nr-cuup', 'nr-cuup', 'oai-nr-cuup-asan', '--build-arg "BUILD_OPTION=--sanitize"'))
405 406
				imageNames.append(('ran-build-fhi72', 'build.fhi72', 'ran-build-fhi72', ''))
				imageNames.append(('oai-gnb', 'gNB.fhi72', 'oai-gnb-fhi72', ''))
407 408
		result = re.search('build_cross_arm64', self.imageKind)
		if result is not None:
409
			self.dockerfileprefix = '.ubuntu22.cross-arm64'
410
		
Raphael Defosseux's avatar
Raphael Defosseux committed
411
		self.testCase_id = HTML.testCase_id
412
		cmd.cd(lSourcePath)
413
		# if asterix, copy the entitlement and subscription manager configurations
414
		if self.host == 'Red Hat':
415
			cmd.run('mkdir -p ./etc-pki-entitlement')
416
			cmd.run('cp /etc/pki/entitlement/*.pem ./etc-pki-entitlement/')
417

418 419 420
		baseImage = 'ran-base'
		baseTag = 'develop'
		forceBaseImageBuild = False
421
		imageTag = 'develop'
422 423
		if (self.ranAllowMerge):
			imageTag = 'ci-temp'
424
			if self.ranTargetBranch == 'develop':
425 426
				cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base{self.dockerfileprefix} | grep --colour=never -i INDEX')
				result = re.search('index', cmd.getBefore())
427
				if result is not None:
428 429
					forceBaseImageBuild = True
					baseTag = 'ci-temp'
430 431 432 433 434
			# if the branch name contains integration_20xx_wyy, let rebuild ran-base
			result = re.search('integration_20([0-9]{2})_w([0-9]{2})', self.ranBranch)
			if not forceBaseImageBuild and result is not None:
				forceBaseImageBuild = True
				baseTag = 'ci-temp'
435
		else:
436
			forceBaseImageBuild = True
437

438
		# Let's remove any previous run artifacts if still there
439
		cmd.run(f"{self.cli} image prune --force")
440 441
		for image,pattern,name,option in imageNames:
			cmd.run(f"{self.cli} image rm {name}:{imageTag}")
442

443 444 445
		# Build the base image only on Push Events (not on Merge Requests)
		# On when the base image docker file is being modified.
		if forceBaseImageBuild:
446
			cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}")
447
			cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> cmake_targets/log/ran-base.log", timeout=1600)
448
		# First verify if the base image was properly created.
449
		ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}")
450
		allImagesSize = {}
451
		if ret.returncode != 0:
452
			logging.error('\u001B[1m Could not build properly ran-base\u001B[0m')
453
			# Recover the name of the failed container?
454
			cmd.run(f"{self.cli} ps --quiet --filter \"status=exited\" -n1 | xargs --no-run-if-empty {self.cli} rm -f")
455 456
			cmd.run(f"{self.cli} image prune --force")
			cmd.close()
457 458 459
			logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
			HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
			HTML.CreateHtmlTabFooter(False)
460
			return False
461
		else:
462
			result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
463 464 465 466 467 468 469 470 471
			if result is not None:
				size = float(result.group("size")) / 1000000
				imageSizeStr = f'{size:.1f}'
				logging.debug(f'\u001B[1m   ran-base size is {imageSizeStr} Mbytes\u001B[0m')
				allImagesSize['ran-base'] = f'{imageSizeStr} Mbytes'
			else:
				logging.debug('ran-base size is unknown')

		# Recover build logs, for the moment only possible when build is successful
472 473 474 475
		cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}")
		cmd.run("mkdir -p cmake_targets/log/ran-base")
		cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base")
		cmd.run(f"{self.cli} rm -f test")
476 477

		# Build the target image(s)
478 479
		status = True
		attemptedImages = ['ran-base']
480 481
		for image,pattern,name,option in imageNames:
			attemptedImages += [name]
482
			# the archived Dockerfiles have "ran-base:latest" as base image
483
			# we need to update them with proper tag
484
			cmd.run(f'git checkout -- docker/Dockerfile.{pattern}{self.dockerfileprefix}')
485
			cmd.run(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
486 487 488
			# target images should use the proper ran-build image
			if image != 'ran-build' and "-asan" in name:
				cmd.run(f'sed -i -e "s#ran-build:latest#ran-build-asan:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
489 490
			elif "fhi72" in name:
				cmd.run(f'sed -i -e "s#ran-build-fhi72:latest#ran-build-fhi72:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
491
			elif image != 'ran-build':
492
				cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
Robert Schmidt's avatar
Robert Schmidt committed
493
			if image == 'oai-gnb-aerial':
494
				cmd.run('cp -f /opt/nvidia-ipc/nvipc_src.2024.05.23.tar.gz .')
495
			ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > cmake_targets/log/{name}.log 2>&1', timeout=1200)
Robert Schmidt's avatar
Robert Schmidt committed
496
			if image == 'oai-gnb-aerial':
497
				cmd.run('rm -f nvipc_src.2024.05.23.tar.gz')
498
			if image == 'ran-build' and ret.returncode == 0:
499 500
				cmd.run(f"docker run --name test-log -d {name}:{imageTag} /bin/true")
				cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/ cmake_targets/log/{name}/")
501 502
				cmd.run(f"docker rm -f test-log")
			else:
503
				cmd.run(f"mkdir -p cmake_targets/log/{name}")
504
			# check the status of the build
505
			ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {name}:{imageTag}")
506
			if ret.returncode != 0:
507
				logging.error('\u001B[1m Could not build properly ' + name + '\u001B[0m')
508
				status = False
509
				# Here we should check if the last container corresponds to a failed command and destroy it
510
				cmd.run(f"{self.cli} ps --quiet --filter \"status=exited\" -n1 | xargs --no-run-if-empty {self.cli} rm -f")
511
				allImagesSize[name] = 'N/A -- Build Failed'
512
				break
513
			else:
514
				result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
515
				if result is not None:
516
					size = float(result.group("size")) / 1000000 # convert to MB
517
					imageSizeStr = f'{size:.1f}'
518 519
					logging.debug(f'\u001B[1m   {name} size is {imageSizeStr} Mbytes\u001B[0m')
					allImagesSize[name] = f'{imageSizeStr} Mbytes'
520
				else:
521 522
					logging.debug(f'{name} size is unknown')
					allImagesSize[name] = 'unknown'
523
			# Now pruning dangling images in between target builds
524
			cmd.run(f"{self.cli} image prune --force")
525

526
		# Remove all intermediate build images and clean up
527
		cmd.run(f"{self.cli} image rm ran-build:{imageTag} ran-build-asan:{imageTag} ran-build-fhi72:{imageTag} || true")
528
		cmd.run(f"{self.cli} volume prune --force")
529 530 531 532 533 534 535

		# Remove some cached artifacts to prevent out of diskspace problem
		logging.debug(cmd.run("df -h").stdout)
		logging.debug(cmd.run("docker system df").stdout)
		cmd.run(f"{self.cli} buildx prune --filter until=1h --force")
		logging.debug(cmd.run("df -h").stdout)
		logging.debug(cmd.run("docker system df").stdout)
536 537 538

		# create a zip with all logs
		build_log_name = f'build_log_{self.testCase_id}'
539 540
		CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
		cmd.close()
541

542
		# Analyze the logs
543
		collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status)
544
		
545 546 547
		if status:
			logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
			HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK)
548
			HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
549
			return True
550 551 552
		else:
			logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
			HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
553
			HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
554
			HTML.CreateHtmlTabFooter(False)
555
			return False
556

557
	def BuildProxy(self, HTML):
558 559
		svr = self.eNB_serverId[self.eNB_instance]
		lIpAddr, lSourcePath = self.GetCredentials(svr)
560
		logging.debug('Building on server: ' + lIpAddr)
561
		ssh = cls_cmd.getConnection(lIpAddr)
562

563
		self.testCase_id = HTML.testCase_id
564 565 566
		oldRanCommidID = self.ranCommitID
		oldRanRepository = self.ranRepository
		oldRanAllowMerge = self.ranAllowMerge
567
		oldRanTargetBranch = self.ranTargetBranch
568 569 570
		self.ranCommitID = self.proxyCommit
		self.ranRepository = 'https://github.com/EpiSci/oai-lte-5g-multi-ue-proxy.git'
		self.ranAllowMerge = False
571
		self.ranTargetBranch = 'master'
572 573

		# Let's remove any previous run artifacts if still there
574
		ssh.run('docker image prune --force')
575
		# Remove any previous proxy image
576
		ssh.run('docker image rm oai-lte-multi-ue-proxy:latest')
577 578 579 580

		tag = self.proxyCommit
		logging.debug('building L2sim proxy image for tag ' + tag)
		# check if the corresponding proxy image with tag exists. If not, build it
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
		ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
		buildProxy = ret.returncode != 0 # if no image, build new proxy
		if buildProxy:
			ssh.run(f'rm -Rf {lSourcePath}')
			success = CreateWorkspace(lIpAddr, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
			if not success:
				raise Exception("could not clone proxy repository")

			filename = f'build_log_{self.testCase_id}'
			fullpath = f'{lSourcePath}/{filename}'

			ssh.run(f'docker build --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file {lSourcePath}/docker/Dockerfile.ubuntu18.04 {lSourcePath} > {fullpath} 2>&1')
			ssh.run(f'zip -r -qq {fullpath}.zip {fullpath}')
			local_file = f"{os.getcwd()}/../cmake_targets/log/{filename}.zip"
			ssh.copyin(f'{fullpath}.zip', local_file)
			# don't delete such that we might recover the zips
			#ssh.run(f'rm -f {fullpath}.zip')

			ssh.run('docker image prune --force')
			ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
601
			if ret.returncode != 0:
602
				logging.error('\u001B[1m Build of L2sim proxy failed\u001B[0m')
603
				ssh.close()
604 605
				HTML.CreateHtmlTestRow('commit ' + tag, 'KO', CONST.ALL_PROCESSES_OK)
				HTML.CreateHtmlTabFooter(False)
606
				return False
607 608 609 610
		else:
			logging.debug('L2sim proxy image for tag ' + tag + ' already exists, skipping build')

		# retag the build images to that we pick it up later
611
		ssh.run(f'docker image tag proxy:{tag} oai-lte-multi-ue-proxy:latest')
612 613 614 615 616 617

		# we assume that the host on which this is built will also run the proxy. The proxy
		# currently requires the following command, and the docker-compose up mechanism of
		# the CI does not allow to run arbitrary commands. Note that the following actually
		# belongs to the deployment, not the build of the proxy...
		logging.warning('the following command belongs to deployment, but no mechanism exists to exec it there!')
618
		ssh.run('sudo ifconfig lo: 127.0.0.2 netmask 255.0.0.0 up')
619

620 621 622 623 624
		# to prevent accidentally overwriting data that might be used later
		self.ranCommitID = oldRanCommidID
		self.ranRepository = oldRanRepository
		self.ranAllowMerge = oldRanAllowMerge
		self.ranTargetBranch = oldRanTargetBranch
625

626 627 628 629 630 631 632 633
		# we do not analyze the logs (we assume the proxy builds fine at this stage),
		# but need to have the following information to correctly display the HTML
		files = {}
		errorandwarnings = {}
		errorandwarnings['errors'] = 0
		errorandwarnings['warnings'] = 0
		errorandwarnings['status'] = True
		files['Target Image Creation'] = errorandwarnings
634 635
		collectInfo = {}
		collectInfo['proxy'] = files
636 637
		ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
		result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', ret.stdout)
638
		# Cleaning any created tmp volume
639
		ssh.run('docker volume prune --force')
640
		ssh.close()
641

642
		allImagesSize = {}
643 644 645
		if result is not None:
			imageSize = float(result.group('size')) / 1000000
			logging.debug('\u001B[1m   proxy size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
646
			allImagesSize['proxy'] = str(round(imageSize,1)) + ' Mbytes'
647 648 649 650
			logging.info('\u001B[1m Building L2sim Proxy Image Pass\u001B[0m')
			HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK)
			HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
			return True
651
		else:
652
			logging.error('proxy size is unknown')
653
			allImagesSize['proxy'] = 'unknown'
654 655 656 657
			logging.error('\u001B[1m Build of L2sim proxy failed\u001B[0m')
			HTML.CreateHtmlTestRow('commit ' + tag, 'KO', CONST.ALL_PROCESSES_OK)
			HTML.CreateHtmlTabFooter(False)
			return False
658

659
	def BuildRunTests(self, HTML):
660 661
		svr = self.eNB_serverId[self.eNB_instance]
		lIpAddr, lSourcePath = self.GetCredentials(svr)
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685
		logging.debug('Building on server: ' + lIpAddr)
		cmd = cls_cmd.RemoteCmd(lIpAddr)
		cmd.cd(lSourcePath)

		ret = cmd.run('hostnamectl')
		result = re.search('Ubuntu', ret.stdout)
		host = result.group(0)
		if host != 'Ubuntu':
			cmd.close()
			raise Exception("Can build unit tests only on Ubuntu server")
		logging.debug('running on Ubuntu as expected')

		if self.forcedWorkspaceCleanup:
			cmd.run(f'sudo -S rm -Rf {lSourcePath}')
		self.testCase_id = HTML.testCase_id
	
		# check that ran-base image exists as we expect it
		baseImage = 'ran-base'
		baseTag = 'develop'
		if self.ranAllowMerge:
			if self.ranTargetBranch == 'develop':
				cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base{self.dockerfileprefix} | grep --colour=never -i INDEX')
				result = re.search('index', cmd.getBefore())
				if result is not None:
686
					baseTag = 'ci-temp'
687 688 689 690 691 692 693 694
		ret = cmd.run(f"docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}")
		if ret.returncode != 0:
			logging.error(f'No {baseImage} image present, cannot build tests')
			HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
			HTML.CreateHtmlTabFooter(False)
			return False

		# build ran-unittests image
695
		dockerfile = "ci-scripts/docker/Dockerfile.unittest.ubuntu22"
696 697
		ret = cmd.run(f'docker build --progress=plain --tag ran-unittests:{baseTag} --file {dockerfile} . &> {lSourcePath}/cmake_targets/log/unittest-build.log')
		if ret.returncode != 0:
698 699
			build_log_name = f'build_log_{self.testCase_id}'
			CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
700 701 702 703 704 705 706 707
			logging.error(f'Cannot build unit tests')
			HTML.CreateHtmlTestRow("Unit test build failed", 'KO', [dockerfile])
			HTML.CreateHtmlTabFooter(False)
			return False

		HTML.CreateHtmlTestRowQueue("Build unit tests", 'OK', [dockerfile])

		# it worked, build and execute tests, and close connection
708
		ret = cmd.run(f'docker run -a STDOUT --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --rm ran-unittests:{baseTag} ctest --output-on-failure --no-label-summary -j$(nproc)')
709
		cmd.run(f'docker rmi ran-unittests:{baseTag}')
710 711 712 713 714 715 716 717 718 719 720 721 722
		build_log_name = f'build_log_{self.testCase_id}'
		CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
		cmd.close()

		if ret.returncode == 0:
			HTML.CreateHtmlTestRowQueue('Unit tests succeeded', 'OK', [ret.stdout])
			HTML.CreateHtmlTabFooter(True)
			return True
		else:
			HTML.CreateHtmlTestRowQueue('Unit tests failed (see also doc/UnitTests.md)', 'KO', [ret.stdout])
			HTML.CreateHtmlTabFooter(False)
			return False

723 724
	def Push_Image_to_Local_Registry(self, HTML, svr_id):
		lIpAddr, lSourcePath = self.GetCredentials(svr_id)
725
		logging.debug('Pushing images from server: ' + lIpAddr)
726
		ssh = cls_cmd.getConnection(lIpAddr)
727
		imagePrefix = 'porcepix.sboai.cs.eurecom.fr'
728 729
		ret = ssh.run(f'docker login -u oaicicd -p oaicicd {imagePrefix}')
		if ret.returncode != 0:
730 731
			msg = 'Could not log into local registry'
			logging.error(msg)
732
			ssh.close()
733
			HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
734 735 736 737 738
			return False

		orgTag = 'develop'
		if self.ranAllowMerge:
			orgTag = 'ci-temp'
739
		for image in IMAGES:
740 741
			tagToUse = CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
			imageTag = f"{image}:{tagToUse}"
742 743
			ret = ssh.run(f'docker image tag {image}:{orgTag} {imagePrefix}/{imageTag}')
			if ret.returncode != 0:
744
				continue
745 746
			ret = ssh.run(f'docker push {imagePrefix}/{imageTag}')
			if ret.returncode != 0:
747
				msg = f'Could not push {image} to local registry : {imageTag}'
748
				logging.error(msg)
749
				ssh.close()
750
				HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
751
				return False
752 753
			# Creating a develop tag on the local private registry
			if not self.ranAllowMerge:
754 755 756 757
				ssh.run(f'docker image tag {image}:{orgTag} {imagePrefix}/{image}:develop')
				ssh.run(f'docker push {imagePrefix}/{image}:develop')
				ssh.run(f'docker rmi {imagePrefix}/{image}:develop')
			ssh.run(f'docker rmi {imagePrefix}/{imageTag} {image}:{orgTag}')
758

759 760
		ret = ssh.run(f'docker logout {imagePrefix}')
		if ret.returncode != 0:
761 762
			msg = 'Could not log off from local registry'
			logging.error(msg)
763
			ssh.close()
764
			HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK)
765 766
			return False

767
		ssh.close()
768
		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
769 770
		return True

771 772 773 774 775 776 777 778
	def Pull_Image(cmd, images, tag, registry, username, password):
		if username is not None and password is not None:
			logging.info(f"logging into registry {username}@{registry}")
			response = cmd.run(f'docker login -u {username} -p {password} {registry}', silent=True, reportNonZero=False)
			if response.returncode != 0:
				msg = f'Could not log into registry {username}@{registry}'
				logging.error(msg)
				return False, msg
779
		pulled_images = []
780 781 782
		for image in images:
			imageTag = f"{image}:{tag}"
			response = cmd.run(f'docker pull {registry}/{imageTag}')
783
			if response.returncode != 0:
784
				msg = f'Could not pull {image} from local registry: {imageTag}'
785
				logging.error(msg)
786 787 788
				return False, msg
			cmd.run(f'docker tag {registry}/{imageTag} oai-ci/{imageTag}')
			cmd.run(f'docker rmi {registry}/{imageTag}')
789
			pulled_images += [f"oai-ci/{imageTag}"]
790 791 792
		if username is not None and password is not None:
			response = cmd.run(f'docker logout {registry}')
			# we have the images, if logout fails it's no problem
793
		msg = "Pulled Images:\n" + '\n'.join(pulled_images)
794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
		return True, msg

	def Pull_Image_from_Registry(self, HTML, svr_id, images, tag=None, registry="porcepix.sboai.cs.eurecom.fr", username="oaicicd", password="oaicicd"):
		lIpAddr, lSourcePath = self.GetCredentials(svr_id)
		logging.debug('\u001B[1m Pulling image(s) on server: ' + lIpAddr + '\u001B[0m')
		if not tag:
			tag = CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
		with cls_cmd.getConnection(lIpAddr) as cmd:
			success, msg = Containerize.Pull_Image(cmd, images, tag, registry, username, password)
		param = f"on node {lIpAddr}"
		if success:
			HTML.CreateHtmlTestRowQueue(param, 'OK', [msg])
		else:
			HTML.CreateHtmlTestRowQueue(param, 'KO', [msg])
		return success
809

810 811 812 813
	def Clean_Test_Server_Images(self, HTML, svr_id, images, tag=None):
		lIpAddr, lSourcePath = self.GetCredentials(svr_id)
		logging.debug(f'\u001B[1m Cleaning image(s) from server: {lIpAddr}\u001B[0m')
		if not tag:
814
			tag = CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
815

816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
		status = True
		with cls_cmd.getConnection(lIpAddr) as myCmd:
			removed_images = []
			for image in images:
				fullImage = f"oai-ci/{image}:{tag}"
				cmd = f'docker rmi {fullImage}'
				if myCmd.run(cmd).returncode != 0:
					status = False
				removed_images += [fullImage]

		msg = "Removed Images:\n" + '\n'.join(removed_images)
		s = 'OK' if status else 'KO'
		param = f"on node {lIpAddr}"
		HTML.CreateHtmlTestRowQueue(param, s, [msg])
		return status
831

832
	def Create_Workspace(self,HTML):
833 834
		svr = self.eNB_serverId[self.eNB_instance]
		lIpAddr, lSourcePath = self.GetCredentials(svr)
835
		success = CreateWorkspace(lIpAddr, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
836
		if success:
837
			HTML.CreateHtmlTestRowQueue('N/A', 'OK', [f"created workspace {lSourcePath}"])
838 839 840
		else:
			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ["cannot create workspace"])
		return success
841

842
	def DeployObject(self, HTML):
843
		svr = self.eNB_serverId[self.eNB_instance]
844
		num_attempts = self.num_attempts
845
		lIpAddr, lSourcePath = self.GetCredentials(svr)
846
		logging.debug(f'Deploying OAI Object on server: {lIpAddr}')
847
		yaml = self.yamlPath[self.eNB_instance].strip('/')
848 849 850
		# creating the log folder by default
		local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}"
		os.system(f'mkdir -p {local_dir}')
851
		wd = f'{lSourcePath}/{yaml}'
852 853
		wd_yaml = f'{wd}/docker-compose.y*ml'
		yaml_dir = yaml.split('/')[-1]
854
		with cls_cmd.getConnection(lIpAddr) as ssh:
855
			services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml)
856
			if services == [] or services == ' ' or services == None:
857
				msg = 'Cannot determine services to start'
858 859 860 861
				logging.error(msg)
				HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg])
				return False
			ExistEnvFilePrint(ssh, wd)
862
			WriteEnvFile(ssh, services, wd, self.deploymentTag, self.flexricTag)
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888
			if num_attempts <= 0:
				raise ValueError(f'Invalid value for num_attempts: {num_attempts}, must be greater than 0')
			for attempt in range(num_attempts):
				imagesInfo = []
				healthInfo = []
				logging.info(f'will start services {services}')
				status = ssh.run(f'docker compose -f {wd_yaml} up -d -- {services}')
				if status.returncode != 0:
					msg = f'cannot deploy services {services}: {status.stdout}'
					logging.error(msg)
					HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [msg])
					return False
				for svc in services.split():
					health, msg = GetServiceHealth(ssh, svc, f'{wd_yaml}')
					logging.info(msg)
					imagesInfo.append(msg)
					healthInfo.append(health)
				deployed = all(healthInfo)
				if deployed:
					break
				elif (attempt < num_attempts - 1):
					logging.warning(f'Failed to deploy on attempt {attempt}, restart services {services}')
					for svc in services.split():
						CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log')
					ssh.run(f'docker compose -f {wd_yaml} down -- {services}')
		if deployed:
889
			HTML.CreateHtmlTestRowQueue('N/A', 'OK', ['\n'.join(imagesInfo)])
Raphael Defosseux's avatar
Raphael Defosseux committed
890
		else:
891
			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)])
892
		return deployed
893

Raphael Defosseux's avatar
Raphael Defosseux committed
894
	def UndeployObject(self, HTML, RAN):
895 896
		svr = self.eNB_serverId[self.eNB_instance]
		lIpAddr, lSourcePath = self.GetCredentials(svr)
897
		logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m')
898 899 900 901 902 903 904 905 906 907
		yaml = self.yamlPath[self.eNB_instance].strip('/')
		wd = f'{lSourcePath}/{yaml}'
		yaml_dir = yaml.split('/')[-1]
		with cls_cmd.getConnection(lIpAddr) as ssh:
			ExistEnvFilePrint(ssh, wd)
			services = GetRunningServices(ssh, f"{wd}/docker-compose.y*ml")
			copyin_res = None
			if services is not None:
				all_serv = " ".join([s for s, _ in services])
				ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}')
908
				copyin_res = all(CopyinServiceLog(ssh, lSourcePath, yaml_dir, s, f"{wd}/docker-compose.y*ml", f'{s}-{HTML.testCase_id}.log') for s, c in services)
909 910 911 912
			else:
				logging.warning('could not identify services to stop => no log file')
			ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v')
			ssh.run(f'rm {wd}/.env')
913
		if not copyin_res:
914
			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)'])
915
			return False
Raphael Defosseux's avatar
Raphael Defosseux committed
916
		else:
917 918
			log_results = [CheckLogs(self, yaml_dir, s, HTML, RAN) for s, _ in services]
			success = all(log_results)
919 920 921 922 923
		if success:
			logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
		else:
			logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m')
		return success