cls_physim1.py 13 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements.  See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# *      contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
#   Required Python Version
#     Python 3.x
#
#   Required Python Package
#     pexpect
#---------------------------------------------------------------------

#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import logging
import os
36 37
import re
import time
38
import subprocess
39
import sys
40 41 42

import sshconnection as SSH
import cls_oai_html
43 44 45 46 47 48 49 50
import constants as CONST
import helpreadme as HELP

class PhySim:
	def __init__(self):
		self.eNBIpAddr = ""
		self.eNBUserName = ""
		self.eNBPassword = ""
Mohammed Ismail's avatar
Mohammed Ismail committed
51 52
		self.OCUserName = ""
		self.OCPassword = ""
53
		self.OCProjectName = ""
54 55 56 57
		self.eNBSourceCodePath = ""
		self.ranRepository = ""
		self.ranBranch = ""
		self.ranCommitID= ""
Mohammed Ismail's avatar
Mohammed Ismail committed
58
		self.ranAllowMerge= False
59
		self.ranTargetBranch= ""
Mohammed Ismail's avatar
Mohammed Ismail committed
60 61
		self.testResult = {}
		self.testSummary = {}
62
		self.testStatus = False
63 64 65 66 67

#-----------------$
#PUBLIC Methods$
#-----------------$

68
	def Deploy_PhySim(self, HTML, RAN):
69 70 71 72 73 74 75
		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
		lIpAddr = self.eNBIPAddress
		lUserName = self.eNBUserName
		lPassWord = self.eNBPassword
		lSourcePath = self.eNBSourceCodePath
Mohammed Ismail's avatar
Mohammed Ismail committed
76 77
		ocUserName = self.OCUserName
		ocPassword = self.OCPassword
78
		ocProjectName = self.OCProjectName
79

80
		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '' or ocUserName == '' or ocPassword == '' or ocProjectName == '':
81 82 83 84 85 86 87 88 89 90 91
			HELP.GenericHelp(CONST.Version)
			sys.exit('Insufficient Parameter')
		logging.debug('Building on server: ' + lIpAddr)
		mySSH = SSH.SSHConnection()
		mySSH.open(lIpAddr, lUserName, lPassWord)

		self.testCase_id = HTML.testCase_id

		# on RedHat/CentOS .git extension is mandatory
		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
		if result is not None:
92
			full_ran_repo_name = self.ranRepository.replace('git/', 'git')
93 94
		else:
			full_ran_repo_name = self.ranRepository + '.git'
95
		mySSH.command('rm -Rf ' + lSourcePath, '\$', 30)
96 97 98 99 100 101 102
		mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
		mySSH.command('cd ' + lSourcePath, '\$', 5)
		mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
		# Raphael: here add a check if git clone or git fetch went smoothly
		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)

103
		mySSH.command('git clean -x -d -ff', '\$', 30)
104 105 106
		mySSH.command('mkdir -p cmake_targets/log', '\$', 5)
		# if the commit ID is provided use it to point to it
		if self.ranCommitID != '':
107
			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
Mohammed Ismail's avatar
Mohammed Ismail committed
108
		if self.ranAllowMerge:
109 110
			branchName = self.ranBranch.replace('/','-')
			imageTag = f'{branchName}-{self.ranCommitID[0:8]}'
111 112
			if self.ranTargetBranch == '':
				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
113
					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30)
114 115
			else:
				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
116
				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 30)
Mohammed Ismail's avatar
Mohammed Ismail committed
117
		else:
118
			imageTag = f'develop-{self.ranCommitID[0:8]}'
119

120
		# logging to OC Cluster and then switch to corresponding project
121
		mySSH.command(f'oc login -u {ocUserName} -p {ocPassword} --server https://api.oai.cs.eurecom.fr:6443', '\$', 30)
122
		if mySSH.getBefore().count('Login successful.') == 0:
Mohammed Ismail's avatar
Mohammed Ismail committed
123
			logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m')
124
			mySSH.close()
125 126 127
			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_LOGIN_FAIL)
			RAN.prematureExit = True
			return
Mohammed Ismail's avatar
Mohammed Ismail committed
128 129
		else:
			logging.debug('\u001B[1m   Login to OC Cluster Successfully\u001B[0m')
130
		mySSH.command(f'oc project {ocProjectName}', '\$', 30)
131 132
		if mySSH.getBefore().count(f'Already on project "{ocProjectName}"') == 0 and mySSH.getBefore().count(f'Now using project "{self.OCProjectName}"') == 0:
			logging.error(f'\u001B[1m Unable to access OC project {ocProjectName}\u001B[0m')
133
			mySSH.command('oc logout', '\$', 30)
134
			mySSH.close()
135 136 137
			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_PROJECT_FAIL)
			RAN.prematureExit = True
			return
Mohammed Ismail's avatar
Mohammed Ismail committed
138
		else:
139
			logging.debug(f'\u001B[1m   Now using project {ocProjectName}\u001B[0m')
Mohammed Ismail's avatar
Mohammed Ismail committed
140

Mohammed Ismail's avatar
Mohammed Ismail committed
141
		# Using helm charts deployment
142
		mySSH.command(f'grep -rl OAICICD_PROJECT ./charts/ | xargs sed -i -e "s#OAICICD_PROJECT#{ocProjectName}#"', '\$', 30)
Mohammed Ismail's avatar
Mohammed Ismail committed
143
		mySSH.command(f'sed -i -e "s#TAG#{imageTag}#g" ./charts/physims/values.yaml', '\$', 6)
sagar arora's avatar
sagar arora committed
144
		mySSH.command('helm install physim ./charts/physims/ --wait 2>&1 | tee -a cmake_targets/log/physim_helm_summary.txt', '\$', 30)
Mohammed Ismail's avatar
Mohammed Ismail committed
145
		if mySSH.getBefore().count('STATUS: deployed') == 0:
Mohammed Ismail's avatar
Mohammed Ismail committed
146
			logging.error('\u001B[1m Deploying PhySim Failed using helm chart on OC Cluster\u001B[0m')
sagar arora's avatar
sagar arora committed
147
			mySSH.command('helm uninstall physim | tee -a cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 30)
148 149 150
			isFinished1 = False
			while(isFinished1 == False):
				time.sleep(20)
151
				mySSH.command('oc get pods -l app=physim', '\$', 6, resync=True)
152 153
				if re.search('No resources found', mySSH.getBefore()):
					isFinished1 = True
154
			mySSH.command('oc logout', '\$', 30)
155
			mySSH.close()
156
			self.AnalyzeLogFile_phySim()
157 158
			RAN.prematureExit = True
			return
Mohammed Ismail's avatar
Mohammed Ismail committed
159 160 161
		else:
			logging.debug('\u001B[1m   Deployed PhySim Successfully using helm chart\u001B[0m')
		isRunning = False
Mohammed Ismail's avatar
Mohammed Ismail committed
162
		count = 0
sagar arora's avatar
sagar arora committed
163
		# Check whether the containers are in Running state or not under 2 mins
164
		while(count < 5 and isRunning == False):
Mohammed Ismail's avatar
Mohammed Ismail committed
165
			time.sleep(60)
166
			mySSH.command('oc get pods -o wide -l app=physim | tee -a cmake_targets/log/physim_pods_summary.txt', '\$', 30, resync=True)
sagar arora's avatar
sagar arora committed
167 168
			running_count = mySSH.getBefore().count('Running')
			completed_count = mySSH.getBefore().count('Completed')
169
			if (running_count + completed_count) == 22:
Mohammed Ismail's avatar
Mohammed Ismail committed
170 171
				logging.debug('\u001B[1m Running the physim test Scenarios\u001B[0m')
				isRunning = True
172
				podNames = re.findall('oai-[\S\d\w]+', mySSH.getBefore())
Mohammed Ismail's avatar
Mohammed Ismail committed
173 174
			count +=1
		if isRunning == False:
sagar arora's avatar
sagar arora committed
175
			logging.error('\u001B[1m Some pods are in Error state \u001B[0m')
176
			mySSH.command('oc get pods -l app=physim 2>&1 | tee -a cmake_targets/log/physim_pods_summary.txt', '\$', 6)
177
			mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc describe pod $pod >> cmake_targets/log/physim_pods_summary.txt; done', '\$', 10)
sagar arora's avatar
sagar arora committed
178
			mySSH.command('helm uninstall physim | tee -a cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6)
179
			self.AnalyzeLogFile_phySim()
180 181 182
			isFinished1 = False
			while(isFinished1 == False):
				time.sleep(20)
183
				mySSH.command('oc get pods -l app=physim', '\$', 6, resync=True)
184 185
				if re.search('No resources found', mySSH.getBefore()):
					isFinished1 = True
186
			mySSH.command('oc logout', '\$', 30)
187 188 189 190
			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_PHYSIM_DEPLOY_FAIL)
			HTML.CreateHtmlTestRowPhySimTestResult(self.testSummary,self.testResult)
			RAN.prematureExit = True
			return
191
		# Waiting to complete the running test
Mohammed Ismail's avatar
Mohammed Ismail committed
192 193
		count = 0
		isFinished = False
194 195
		# doing a deep copy!
		tmpPodNames = podNames.copy()
196
		while(count < 9 and isFinished == False):
197
			time.sleep(60)
Mohammed Ismail's avatar
Mohammed Ismail committed
198
			for podName in tmpPodNames:
199 200
				mySSH.command2(f'oc logs --tail=1 {podName} 2>&1', 6, silent=True)
				if mySSH.cmd2Results.count('FINISHED') != 0:
201
					logging.debug(podName + ' is finished')
Mohammed Ismail's avatar
Mohammed Ismail committed
202 203
					tmpPodNames.remove(podName)
			if not tmpPodNames:
204
				isFinished = True
Mohammed Ismail's avatar
Mohammed Ismail committed
205 206 207
			count += 1
		if isFinished:
			logging.debug('\u001B[1m PhySim test is Complete\u001B[0m')
208 209
		else:
			logging.error('\u001B[1m PhySim test Timed-out!\u001B[0m')
210

211 212
		# Getting the logs of each executables running in individual pods
		for podName in podNames:
213
			mySSH.command(f'oc logs {podName} >> cmake_targets/log/physim_test.txt 2>&1', '\$', 15, resync=True)
214
		mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc describe pod $pod >> cmake_targets/log/physim_pods_summary.txt; done', '\$', 10)
215 216 217 218 219
		mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/log/physim_test.txt', '.')
		try:
			listLogFiles =  subprocess.check_output('egrep --colour=never "Execution Log file|Linux oai-" physim_test.txt', shell=True, universal_newlines=True)
			for line in listLogFiles.split('\n'):
				res1 = re.search('Linux (?P<pod>oai-[a-zA-Z0-9\-]+) ', str(line))
220
				res2 = re.search('Execution Log file = (?P<name>[a-zA-Z0-9\-\/\.\_\+]+)', str(line))
221 222 223 224
				if res1 is not None:
					podName = res1.group('pod')
				if res2 is not None:
					logFileInPod = res2.group('name')
225
					folderName = logFileInPod.replace('/opt/oai-physim/cmake_targets/autotests/log/', '')
226
					folderName = re.sub('/test.*', '', folderName)
227
					fileName = logFileInPod.replace('/opt/oai-physim/cmake_targets/autotests/log/' + folderName + '/', '')
228 229 230 231
					mySSH.command('mkdir -p cmake_targets/log/' + folderName, '\$', 5, silent=True)
					mySSH.command('oc cp ' + podName + ':' + logFileInPod + ' cmake_targets/log/' + folderName + '/' + fileName, '\$', 20, silent=True)
		except Exception as e:
			pass
Mohammed Ismail's avatar
Mohammed Ismail committed
232

233
		# UnDeploy the physical simulator pods
Mohammed Ismail's avatar
Mohammed Ismail committed
234
		mySSH.command('helm uninstall physim | tee -a cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6)
235 236
		isFinished1 = False
		while(isFinished1 == False):
237
			time.sleep(20)
238
			mySSH.command('oc get pods -l app=physim', '\$', 6, resync=True)
239 240 241
			if re.search('No resources found', mySSH.getBefore()):
				isFinished1 = True
		if isFinished1 == True:
Mohammed Ismail's avatar
Mohammed Ismail committed
242
			logging.debug('\u001B[1m UnDeployed PhySim Successfully on OC Cluster\u001B[0m')
Mohammed Ismail's avatar
Mohammed Ismail committed
243 244
		mySSH.command('oc logout', '\$', 6)
		mySSH.close()
245
		self.AnalyzeLogFile_phySim()
246
		if self.testStatus and isFinished:
247 248 249 250 251
			HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
			HTML.CreateHtmlTestRowPhySimTestResult(self.testSummary,self.testResult)
			logging.info('\u001B[1m Physical Simulator Pass\u001B[0m')
		else:
			RAN.prematureExit = True
252 253 254 255
			if isFinished:
				HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK)
			else:
				HTML.CreateHtmlTestRow('Some test(s) timed-out!', 'KO', CONST.ALL_PROCESSES_OK)
256
			HTML.CreateHtmlTestRowPhySimTestResult(self.testSummary,self.testResult)
257
			logging.error('\u001B[1m Physical Simulator Fail\u001B[0m')
Mohammed Ismail's avatar
Mohammed Ismail committed
258

259
	def AnalyzeLogFile_phySim(self):
Mohammed Ismail's avatar
Mohammed Ismail committed
260
		mySSH = SSH.SSHConnection()
261 262 263 264
		mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
		dirToCopy = f'{self.eNBSourceCodePath}/cmake_targets/log/'
		mySSH.copyin(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, dirToCopy, f'./physim_test_logs_{self.testCase_id}/')
		mySSH.command(f'rm -rf {dirToCopy}', '\$', 5)
265
		mySSH.close()
Mohammed Ismail's avatar
Mohammed Ismail committed
266 267
		# physim test log analysis
		nextt = 0
268 269
		nbTests = 0
		nbFailed = 0
Mohammed Ismail's avatar
Mohammed Ismail committed
270 271
		if (os.path.isfile(f'./physim_test_logs_{self.testCase_id}/physim_test.txt')):
			with open(f'./physim_test_logs_{self.testCase_id}/physim_test.txt', 'r') as logfile:
272
				for line in logfile:
273 274 275
					# the following regex would match the following line:
					# execution nr_pbchsim.106rb.test1 {Test1: PBCH-only, 106 PRB} Run_Result = " Run_1 =PASS Run_2 =PASS Run_3 =PASS"  Result = PASS
					#           ^testname               ^testdescription                                 ^test1      ^test2      ^test3          ^status
276
					ret = re.search('execution\s+(?P<name>[a-zA-Z0-9\._\-\+]+)\s+{(?P<desc>[A-Za-z0-9\.\_\-\+:,;\/\%\=\(\)\s]+)}\s+Run_Result\s*=\s*\"\s*Run_1\s*=\s*(?P<run1>[A-Za-z]+)\s*Run_2\s*=\s*(?P<run2>[A-Za-z]+)\s*Run_3\s*=\s*(?P<run3>[A-Za-z]+)\s*\"\s+Result\s*=\s*(?P<status>[A-Za-z]+)', line)
277 278 279 280 281 282 283 284 285 286 287 288 289
					if ret is None:
						continue
					nbTests += 1
					r = [ret.group('run1'), ret.group('run2'), ret.group('run3')]
					nbPass = sum([x == 'PASS' for x in r])
					resultstr = 'PASS'
					if nbPass < 3 or ret.group('status') != 'PASS':
						resultstr = f'FAIL ({3-nbPass}/3)'
						nbFailed += 1
					self.testResult[ret.group('name')] = [ret.group('desc'), resultstr]
		self.testSummary['Nbtests'] = nbTests
		self.testSummary['Nbpass'] =  nbTests - nbFailed
		self.testSummary['Nbfail'] =  nbFailed
290 291
		if self.testSummary['Nbfail'] == 0:
			self.testStatus = True
292
		return 0