CI: Add functions for OC Physim deployment and analysis

ci-scripts/cls_analysis.py

@@ -20,8 +20,11 @@
 # */
 #---------------------------------------------------------------------
 
-import logging
 import re
+import os
+
+import xml.etree.ElementTree as ET
+import json
 
 # Define the mapping of physim_test values to search patterns
 PHYSIM_PATTERN_MAPPING = {
@@ -80,3 +83,65 @@ class Analysis():
 			msg = f'{time1_match.group(1)}: {time1_match.group(2)} us\n{time2_match.group(1)}: {time2_match.group(2)} us exceeds a limit of {threshold} us'
 
 		return success,msg
+
+	def analyze_oc_physim(result_junit, details_json):
+		try:
+			tree = ET.parse(result_junit)
+			root = tree.getroot()
+			nb_tests = int(root.attrib["tests"])
+			nb_failed = int(root.attrib["failures"])
+		except ET.ParseError as e:
+			return False, False, f'Could not parse XML log file {result_junit}: {e}'
+		except FileNotFoundError as e:
+			return False, False, f'JUnit XML log file {result_junit} not found: {e}'
+		except Exception as e:
+			return False, False, f'While parsing JUnit XML log file: exception: {e}'
+
+		try:
+			with open(details_json) as f:
+				j = json.load(f)
+			# prepare JSON for easier access of strings
+			json_test_desc = {}
+			for e in j["tests"]:
+				json_test_desc[e["name"]] = e
+
+		except json.JSONDecodeError as e:
+			return False, False, f'Could not decode JSON log file {details_json}: {e}'
+		except FileNotFoundError as e:
+			return False, False, f'Physim JSON log file {details_json} not found: {e}'
+		except Exception as e:
+			return False, False, f'While parsing physim JSON log file: exception: {e}'
+
+		test_result = {}
+		for test in root: # for each test
+			test_name = test.attrib["name"]
+			test_exec = json_test_desc[test_name]["properties"][1]["value"][0]
+			desc = json_test_desc[test_name]["properties"][1]["value"][1]
+			# get runtime and checks
+			test_check = test.attrib["status"] == "run"
+			time = round(float(test.attrib["time"]), 1)
+			time_check = time < 150
+			output = test.findtext("system-out")
+			output_check = "exceeds the threshold" not in output
+			# collect logs
+			log_dir = f'../cmake_targets/log/{test_exec}'
+			os.makedirs(log_dir, exist_ok=True)
+			with open(f'{log_dir}/{test_name}.log', 'w') as f:
+				f.write(output)
+			# prepare result and info
+			info = f"Runtime: {f'{time:.3f}'[:5]} s"
+			resultstr = 'PASS' if (test_check and time_check and output_check) else 'FAIL'
+			if test_check:
+				if not output_check:
+					info += " Test log exceeds maximal allowed length 100 kB"
+				if not time_check:
+					info += " Test exceeds 150s"
+				if not (time_check and output_check):
+					nb_failed += 1 # time threshold/output length error, not counted for by ctest as of now
+			test_result[test_name] = [desc, info, resultstr]
+
+		test_summary = {}
+		test_summary['Nbtests'] = nb_tests
+		test_summary['Nbpass'] =  nb_tests - nb_failed
+		test_summary['Nbfail'] =  nb_failed
+		return nb_failed == 0, test_summary, test_result

ci-scripts/cls_cluster.py

@@ -31,8 +31,10 @@
 import logging
 import re
 import time
+import os
 
 import cls_oai_html
+import cls_analysis
 import constants as CONST
 import helpreadme as HELP
 import cls_containerize
@@ -457,3 +459,38 @@ class Cluster:
 		HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize)
 
 		return status
+
+	def deploy_oc_physim(self, HTML, oc_release, svr_id):
+		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
+			HELP.GenericHelp(CONST.Version)
+			raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {self.ranBranch} ranCommitID {self.ranCommitID}')
+		image_tag = cls_containerize.CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
+		logging.debug(f'Running physims from server: {svr_id}')
+		script = "scripts/oc-deploy-physims.sh"
+		options = f"oaicicd-core-for-ci-ran {oc_release} {image_tag} {self.eNBSourceCodePath}"
+		ret = cls_cmd.runScript(svr_id, script, 600, options)
+		logging.debug(f'"{script}" finished with code {ret.returncode}, output:\n{ret.stdout}')
+		log_dir = f'{os.getcwd()}/../cmake_targets/log'
+		os.makedirs(log_dir, exist_ok=True)
+		result_junit = f'{oc_release}-run.xml'
+		details_json = f'{oc_release}-tests.json'
+		with cls_cmd.getConnection(svr_id) as ssh:
+			ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{details_json}', tgt=f'{log_dir}/{details_json}')
+			ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{result_junit}', tgt=f'{log_dir}/{result_junit}')
+			ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt', tgt=f'{log_dir}/physim_log.txt')
+			ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt', tgt=f'{log_dir}/physim_pods_summary.txt')
+			ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log', tgt=f'{log_dir}/LastTestsFailed.log')
+		test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(f'{log_dir}/{result_junit}', f'{log_dir}/{details_json}')
+		if test_summary:
+			if test_status:
+				HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
+				HTML.CreateHtmlTestRowPhySimTestResult(test_summary, test_result)
+				logging.info('\u001B[1m Physical Simulator Pass\u001B[0m')
+			else:
+				HTML.CreateHtmlTestRow('Some test(s) failed!', 'KO', CONST.OC_PHYSIM_DEPLOY_FAIL)
+				HTML.CreateHtmlTestRowPhySimTestResult(test_summary, test_result)
+				logging.error('\u001B[1m Physical Simulator Fail\u001B[0m')
+		else:
+			HTML.CreateHtmlTestRowQueue('Physical simulator failed', 'KO', [test_result])
+			logging.error('\u001B[1m Physical Simulator Fail\u001B[0m')
+		return test_status

ci-scripts/cls_oai_html.py

@@ -564,13 +564,13 @@ class HTMLManagement():
 			self.htmlFile.write('        <th>Nb Pass</th>\n')
 			self.htmlFile.write('      </tr>\n')
 			self.htmlFile.write('      <tr>\n')
-			self.htmlFile.write('        <td colspan="2" bgcolor = "lightcyan" > physim_test.txt  </td>\n')
+			self.htmlFile.write('        <td colspan="2" bgcolor = "lightcyan" > physim_log.txt  </td>\n')
 			self.htmlFile.write('        <td colspan="2" bgcolor = "lightcyan" >' + str(testSummary['Nbtests']) + ' </td>\n')
 			if testSummary['Nbfail'] == 0:
-				self.htmlFile.write('        <td bgcolor = "lightcyan" >' + str(testSummary['Nbfail']) + ' </td>\n')
+				self.htmlFile.write('        <td bgcolor = "lightcyan" >' + str(testSummary['Nbfail']) + '</td>\n')
 			else:
-				self.htmlFile.write('        <td bgcolor = "red" >' + str(testSummary['Nbfail']) + ' </td>\n')
-			self.htmlFile.write('        <td gcolor = "lightcyan" >' + str(testSummary['Nbpass']) + ' </td>\n')
+				self.htmlFile.write('        <td bgcolor = "red" ><font color="white">' + str(testSummary['Nbfail']) + '</font></td>\n')
+			self.htmlFile.write('        <td bgcolor = "lightcyan" >' + str(testSummary['Nbpass']) + ' </td>\n')
 			self.htmlFile.write('      </tr>\n')
 			self.htmlFile.write('      <tr bgcolor = "#F0F0F0" >\n')
 			self.htmlFile.write('        <td colspan="6"><b> ---- PHYSIM TEST DETAIL INFO---- </b></td>\n')
@@ -578,22 +578,24 @@ class HTMLManagement():
 			self.htmlFile.write('      <tr bgcolor = "#33CCFF" >\n')
 			self.htmlFile.write('        <th colspan="2">Test Name</th>\n')
 			self.htmlFile.write('        <th colspan="2">Test Description</th>\n')
-			self.htmlFile.write('        <th colspan="2">Result</th>\n')
+			self.htmlFile.write('        <th>Test Status</th>\n')
+			self.htmlFile.write('        <th>Info</th>\n')
 			self.htmlFile.write('      </tr>\n')
 			y = ''
 			for key, value in testResult.items():
 				x = key.split(".")
-				if x[0] != y:
+				if x[2] != y:
 					self.htmlFile.write('      <tr bgcolor = "lightgreen" >\n')
-					self.htmlFile.write('        <td style="text-align: center;" colspan="6"><b>"' + x[0] + '" series </b></td>\n')
+					self.htmlFile.write('        <td style="text-align: center;" colspan="6"><b>"' + x[2] + '" series </b></td>\n')
 					self.htmlFile.write('      </tr>\n')
-					y = x[0]
+					y = x[2]
 				self.htmlFile.write('      <tr>\n')
 				self.htmlFile.write('        <td colspan="2" bgcolor = "lightcyan" >' + key  + ' </td>\n')
 				self.htmlFile.write('        <td colspan="2" bgcolor = "lightcyan" >' + value[0]  + '</td>\n')
 				if 'PASS' in value:
-					self.htmlFile.write('        <td colspan="2" bgcolor = "green" >' + value[1]  + '</td>\n')
+					self.htmlFile.write('        <td bgcolor = "green" ><font color="white"><b>' + value[2]  + '</b></font></td>\n')
 				else:
-					self.htmlFile.write('        <td colspan="2" bgcolor = "red" >' + value[1]  + '</td>\n')
+					self.htmlFile.write('        <td bgcolor = "red" ><font color="white"><b>' + value[2]  + '</b></font></td>\n')
+				self.htmlFile.write('        <td bgcolor = "lightcyan">' + value[1] + '</td>\n')
 
 		self.htmlFile.close()

ci-scripts/scripts/oc-deploy-physims.sh

+#!/bin/bash
+
+set -e
+
+function die() { echo $@; exit 1; }
+[ $# -eq 4 ] || die "usage: $0 <namespace> <release> <image tag> <oai directory>"
+
+OC_NS=${1}
+OC_RELEASE=${2}
+IMG_TAG=${3}
+OAI_DIR=${4}
+
+cat /opt/oc-password | oc login -u oaicicd --server https://api.oai.cs.eurecom.fr:6443 > /dev/null
+oc project ${OC_NS} > /dev/null
+oc tag oaicicd-ran/oai-physim:${IMG_TAG} ${OC_NS}/oai-physim:${IMG_TAG}
+helm install ${OC_RELEASE} ${OAI_DIR}/charts/${OC_RELEASE} --set global.image.version=${IMG_TAG} --wait
+POD_ID=$(oc get pods | grep oai-${OC_RELEASE} | awk '{print $1}')
+sleep 10
+echo "Monitoring logs for 'FINISHED' in pod '$POD_ID'"
+oc logs -f -n ${OC_NS} "$POD_ID" | while read -r line; do
+  if [[ "$line" == *"FINISHED"* ]]; then
+    echo "'FINISHED' detected in logs. Copying logs..."
+    oc logs -n ${OC_NS} "$POD_ID" >> ${OAI_DIR}/ci-scripts/physim_log.txt
+    oc describe pod $POD_ID >> ${OAI_DIR}/ci-scripts/physim_pods_summary.txt
+    oc cp "$POD_ID":/opt/oai-physim/Testing/Temporary/LastTestsFailed.log ${OAI_DIR}/ci-scripts/LastTestsFailed.log
+    oc cp "$POD_ID":/opt/oai-physim/${OC_RELEASE}-tests.json ${OAI_DIR}/ci-scripts/${OC_RELEASE}-tests.json
+    oc cp "$POD_ID":/opt/oai-physim/${OC_RELEASE}-run.xml ${OAI_DIR}/ci-scripts/${OC_RELEASE}-run.xml
+    break
+  fi
+done
+helm uninstall ${OC_RELEASE} --wait
+oc delete istag oai-physim:${IMG_TAG} -n ${OC_NS}
+oc logout > /dev/null