Commit d4e13596 authored by hardy's avatar hardy

major update / refactoring for ran dashboard, including mysql db and test results table

parent 0aa06742
Branches unavailable
2025.w12 2025.w11 2025.w10 2025.w09 2025.w08 2025.w07 2025.w06 2025.w05 2025.w04 2025.w03 2025.w02 2024.w51 2024.w50 2024.w49 2024.w48 2024.w47 2024.w46 2024.w45 2024.w44 2024.w43 2024.w42 2024.w41 2024.w40 2024.w39 2024.w38 2024.w36 2024.w35 2024.w34 2024.w33 2024.w32 2024.w31 2024.w30 2024.w29 2024.w28 2024.w27 2024.w26 2024.w25 2024.w24 2024.w23 2024.w22 2024.w21 2024.w18 2024.w17 2024.w16 2024.w15 2024.w14 2024.w13 2024.w12 2024.w11 2024.w10 2024.w09 2024.w08 2024.w06 2024.w05 2024.w04 2024.w03 2024.w02 2024.w01 2023.w51 2023.w50 2023.w49 2023.w48 2023.w47 2023.w45 2023.w43 2023.w42 2023.w41 2023.w40 2023.w39 2023.w38 2023.w37 2023.w36 2023.w34 2023.w33 2023.w32 2023.w31 2023.w30 2023.w29 2023.w28 2023.w27 2023.w26 2023.w25 2023.w24 2023.w23 2023.w22 2023.w21 2023.w20 2023.w19 2023.w18 2023.w18b 2023.w16 2023.w15 2023.w14 2023.w13 2023.w12 2023.w11 2023.w11b 2023.w10 2023.w10b 2023.w09 2023.w08 2023.w08b 2023.w07 2023.w06 2023.w05 2023.w03 2023.w02 2022.42 2022.41 2022.w51 2022.w50 2022.w49 2022.w48 2022.w47 2022.w46 2022.w45 2022.w43 2022.w42 2022.w42b 2022.w41 2022.w40 2022.w39 2022.w38 2022.w37 2022.w37b 2022.w36 2022.w35 2022.w33 2022.w32 2022.w31 2022.w31b 2022.w30 2022.w29 2022.w26 2022.w25 2022.w24 2022.w24b 2022.w23 2022.w22 2022.w21 2022.w20 2022.w19 2022.w18 2022.w17 2022.w15 2022.w15b 2022.w14a 2022.w13 2022.w13b 2022.w13a 2022.w12 2022.w10 2022.w09 2022.w09b 2022.w08 2022.w08b 2022.w07 2022.w07b 2022.w06 2022.w06a 2022.w05 2022.w05b 2022.w03_hotfix 2022.w03_b 2022.w02 2022.w01 2021.wk46 2021.w51_c 2021.w51_a 2021.w50_a 2021.w49_b 2021.w49_a 2021.w48 2021.w47 2021.w46 2021.w46-powder v2.2.0 v2.1.0 v2.0.0 setparam flexran-eol ARC_1.3
No related merge requests found
version: '2'
services:
mysql:
container_name: oaicicd_mysql
restart: always
image: mysql:latest
environment:
MYSQL_ROOT_PASSWORD: 'ucZBc2XRYdvEm59F'
ports:
- "3307:3306"
volumes:
- /home/oaicicd/mysql/data:/var/lib/mysql
import pymysql
import sys
from datetime import datetime
#This is the script used to write the test results to the mysql DB
#Called by Jenkins pipeline (jenkinsfile)
#Must be located in /home/oaicicid/mysql on the database host
#Usage from Jenkinsfile :
#python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} ''
class SQLConnect:
def __init__(self):
self.connection = pymysql.connect(
host='172.22.0.2',
user='root',
password = 'ucZBc2XRYdvEm59F',
db='oaicicd_tests',
port=3306
)
def put(self,TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS):
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
cur=self.connection.cursor()
cur.execute ('INSERT INTO test_results (TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS,DATE) VALUES (%s,%s,%s,%s,%s,%s,%s);' , (TEST, MR, BRANCH, BUILD, BUILD_LINK, STATUS, now))
self.connection.commit()
self.connection.close()
if __name__ == "__main__":
mydb=SQLConnect()
mydb.put(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6])
......@@ -42,6 +42,11 @@ import datetime #now() and date formating
from datetime import datetime
import re
import gitlab
import yaml
import os
import pickle
from sqlconnect import SQLConnect
#-----------------------------------------------------------
# Class Declaration
......@@ -59,57 +64,75 @@ class gDashboard:
self.ss.del_worksheet(self.sheet) #start by deleting the old sheet
self.sheet = self.ss.add_worksheet(title=worksheet, rows="100", cols="30") #create a new one
self.d = {} #data dictionary
#init with data sources : git, yaml config file, test results databases
cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """
self.git = self.__getGitData(cmd) #git data from Gitlab
self.tests = self.__loadCfg('ran_dashboard_cfg.yaml') #tests table setup from yaml
self.db = self.__loadFromDB() #test results from database
def __loadCfg(self,yaml_file):
with open(yaml_file,'r') as f:
tests = yaml.load(f)
return tests
def fetchData(self,cmd):
def __getGitData(self,cmd):
#cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """
process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
output = process.stdout.readline()
tmp=output.decode("utf-8")
self.d = json.loads(tmp)
d = json.loads(tmp)
return d
def __loadFromDB(self):
mr_list=[]
for x in range(len(self.git)):
mr_list.append(str(self.git[x]['iid']))
mydb=SQLConnect()
for MR in mr_list:
mydb.get(MR)
mydb.close_connection()
return mydb.data
def gBuild(self, destinationSheetName):
#line 1 : update date/time, format dd/mm/YY H:M:S
###line 1 : update date/time, format dd/mm/YY H:M:S
now = datetime.now()
dt_string = "Update : " + now.strftime("%d/%m/%Y %H:%M")
row =[dt_string]
self.sheet.insert_row(row, index=1, value_input_option='RAW')
#line 2 is for the test names (links to jenkins pipeline), updated at the end
###line 2 is for the test short names (links to jenkins pipeline), updated at the end
#line 3 is for the column names
###line 3 is for the column names
i=3
row =["MR","Created_at","Author","Title","Assignee", "Reviewer", "CAN START","IN PROGRESS","COMPLETED","Review Form","OK MERGE","Merge conflicts"]
#test 1
row.append("# PASS")
row.append("# FAIL")
row.append("Last Fail")
#test 2
#tests
for t in range(0,len(self.tests)):
row.append("# PASS")
row.append("# FAIL")
row.append("Last Fail")
self.sheet.insert_row(row, index=i, value_input_option='RAW')
#line 4 onward, MR data lines
for x in range(len(self.d)):
###line 4 onward, MR data lines
for x in range(len(self.git)):
i=i+1
date_time_str = self.d[x]['created_at']
date_time_str = self.git[x]['created_at']
date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ')
milestone1=milestone2=milestone3=milestone4=""
if self.d[x]['milestone']!=None:
if self.d[x]['milestone']['title']=="REVIEW_CAN_START":
if self.git[x]['milestone']!=None:
if self.git[x]['milestone']['title']=="REVIEW_CAN_START":
milestone1="X"
elif self.d[x]['milestone']['title']=="REVIEW_IN_PROGRESS":
elif self.git[x]['milestone']['title']=="REVIEW_IN_PROGRESS":
milestone2="X"
elif self.d[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED":
elif self.git[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED":
milestone3="X"
elif self.d[x]['milestone']['title']=="OK_TO_BE_MERGED":
elif self.git[x]['milestone']['title']=="OK_TO_BE_MERGED":
milestone4="X"
else:
pass
......@@ -117,18 +140,18 @@ class gDashboard:
pass
#check if empty or not
if self.d[x]['assignee']!=None:
assignee = str(self.d[x]['assignee']['name'])
if self.git[x]['assignee']!=None:
assignee = str(self.git[x]['assignee']['name'])
else:
assignee = ""
#check if empty or not
if len(self.d[x]['reviewers'])!=0:
reviewer = str(self.d[x]['reviewers'][0]['name'])
if len(self.git[x]['reviewers'])!=0:
reviewer = str(self.git[x]['reviewers'][0]['name'])
else:
reviewer = ""
if self.d[x]['has_conflicts']==True:
if self.git[x]['has_conflicts']==True:
conflicts = "YES"
else:
conflicts = ""
......@@ -141,12 +164,12 @@ class gDashboard:
project = gl.projects.get(project_id)
#get the opened MR in the project
mrs = project.mergerequests.list(state='opened')
review_form=''
for m in range (0,len(mrs)):
if mrs[m].iid==self.d[x]['iid']:#check the iid is the one we are on
if mrs[m].iid==self.git[x]['iid']:#check the iid is the one we are on
mr_notes = mrs[m].notes.list(all=True)
n=0
found=False
review_form=""
while found==False and n<len(mr_notes):
res=re.search('Code Review by',mr_notes[n].body)#this is the marker we are looking for in all notes
if res!=None:
......@@ -154,69 +177,71 @@ class gDashboard:
found=True
n+=1
#add test results, notice : NSA only for the moment (code to be refactored for multiple tests afterwards)
cmd="python3 /home/oaicicd/mysql/sql_connect_dev.py "+str(self.d[x]['iid'])
process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
output = process.stdout.readline()
tmp=output.decode("utf-8")
tmp=tmp.replace('\'','"')
tmp=tmp.rstrip()
testres = json.loads(tmp)
#build final row to be inserted, the first column is left empty for now, will be filled afterward with hyperlinks to gitlab MR
row =["", str(date_time_obj.date()),str(self.d[x]['author']['name']),str(self.d[x]['title']),\
row =["", str(date_time_obj.date()),str(self.git[x]['author']['name']), str(self.git[x]['title']),\
assignee, reviewer,\
milestone1,milestone2,milestone3,review_form,milestone4,conflicts,\
testres['PASS'],testres['FAIL']]
milestone1,milestone2,milestone3,review_form,milestone4,conflicts]
#and append the test results coming from self.db
mr=str(self.git[x]['iid'])
for t in self.tests:
if mr in self.db:
job=self.tests[t]['job']
if job in self.db[mr]:
if 'PASS' in self.db[mr][job]:
row.append(self.db[mr][job]['PASS'])
else:
row.append('')
if 'FAIL' in self.db[mr][job]:
row.append(self.db[mr][job]['FAIL'])
else:
row.append('')
row.append('')
else:
row.append('')
row.append('')
row.append('')
#insert the row to worksheet
#insert the final row to worksheet
self.sheet.insert_row(row, index=i, value_input_option='RAW')
#add MR hyperlinks in a list of requests to be sent as one update batch; this to save API calls (quotas)
i=3
requests=[]
for x in range(len(self.d)):
for x in range(len(self.git)):
rowIndex=i
colIndex=0
hyperlink= '\"'+"https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/"+ str(self.d[x]['iid']) +'\"'
text= '\"'+str(self.d[x]['iid'])+'"'
hyperlink= '\"'+"https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/"+ str(self.git[x]['iid']) +'\"'
text= '\"'+str(self.git[x]['iid'])+'"'
requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex))
cmd="python3 /home/oaicicd/mysql/sql_connect_dev.py "+str(self.d[x]['iid'])
process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
output = process.stdout.readline()
tmp=output.decode("utf-8")
tmp=tmp.replace('\'','"')
tmp=tmp.rstrip()
testres = json.loads(tmp)
if len(testres['fails list'])>0:
mr=str(self.git[x]['iid'])
colIndex=14
hyperlink= '\"'+ testres['fails link'][0] +'\"'
text= '\"'+testres['fails list'][0]+'"'
for t in self.tests:
job=self.tests[t]['job']
if job in self.db[mr]:
if len(self.db[mr][job]['last_fail'])>0:
hyperlink= '\"'+ self.db[mr][job]['last_fail'][1] +'\"'
text= '\"'+self.db[mr][job]['last_fail'][0]+'"'
requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex))
i=i+1
colIndex+=3
i=i+1 #increment row index for next MR
body = {"requests": requests}
self.ss.batch_update(body)
#line 2 is for the test names
###line 2 is for the test names
#add MR hyperlinks in a list of requests to be sent as one update batch; this to save API calls (quotas)
requests=[]
rowIndex=1
colIndex=12
hyperlink= '\"'+"https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-NSA-Mini-Module/"+'\"'
text= '\"'+'NSA Mini'+'"'
requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex))
rowIndex=1
colIndex=15
hyperlink= '\"'+"https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-SA-Module/"+'\"'
text= '\"'+'NR SA'+'"'
requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex))
for t in self.tests :
hyperlink= '\"'+self.tests[t]['link']+'\"'
short_name= '\"'+ t +'\"'
requests.append(self.addHyperlink(hyperlink, short_name, destinationSheetName, rowIndex, colIndex))
colIndex+=3
body = {"requests": requests}
self.ss.batch_update(body)
......@@ -264,14 +289,14 @@ class gDashboard:
"startRowIndex": 1,
"endRowIndex": 40,
"startColumnIndex": 0,
"endColumnIndex": 19
"endColumnIndex": 26
},
"destination": {
"sheetId": destinationSheetId,
"startRowIndex": 1,
"endRowIndex": 40,
"startColumnIndex": 0,
"endColumnIndex": 19
"endColumnIndex": 26
},
"pasteType": "PASTE_FORMAT"
}
......@@ -358,8 +383,6 @@ class gDashboard:
def main():
my_gDashboard=gDashboard("/opt/dashboard/g_creds.json", 'OAI RAN Dashboard', 'MR Status')
cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """
my_gDashboard.fetchData(cmd)
my_gDashboard.gBuild("MR Status")
my_gDashboard.gFormat("Formating" , "MR Status")
......
import pymysql
import sys
from datetime import datetime
import pickle
#This is the script/package used by the dashboard to retrieve the MR test results from the database
class SQLConnect:
def __init__(self):
self.connection = pymysql.connect(
host='172.22.0.2',
user='root',
password = 'ucZBc2XRYdvEm59F',
db='oaicicd_tests',
port=3306
)
self.data={}
#retrieve data from mysql database and organize it in a dictionary (per MR passed as argument)
def get(self,MR):
self.data[MR]={}
cur=self.connection.cursor()
#get counters per test
sql = "select TEST,STATUS, count(*) AS COUNT from test_results where MR=(%s) group by TEST, STATUS;"
cur.execute(sql,MR)
response=cur.fetchall()
if len(response)==0:#no test results yet
self.data[MR]['PASS']=''
self.data[MR]['FAIL']=''
else:
for i in range(0,len(response)):
test=response[i][0]
status=response[i][1]
count=response[i][2]
if test in self.data[MR]:
self.data[MR][test][status]=count
else:
self.data[MR][test]={}
self.data[MR][test][status]=count
#get last failing build and link
sql = "select TEST,BUILD, BUILD_LINK from test_results where MR=(%s) and STATUS='FAIL' order by DATE DESC;"
cur.execute(sql,MR)
response=cur.fetchall()
if len(response)!=0:
for i in range(0,len(response)):
test=response[i][0]
build=response[i][1]
link=response[i][2]
if 'last_fail' not in self.data[MR][test]:
self.data[MR][test]['last_fail']=[]
self.data[MR][test]['last_fail'].append(build)
self.data[MR][test]['last_fail'].append(link)
#close database connection
def close_connection(self):
self.connection.close()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment