Commit 99e9d338 authored by hardy's avatar hardy

Merge remote-tracking branch 'origin/ci_bring_test_results_to_dashboard' into...

Merge remote-tracking branch 'origin/ci_bring_test_results_to_dashboard' into integration_2021_wk46_c
parents 04aea204 b16d5a6f
...@@ -33,9 +33,12 @@ pipeline { ...@@ -33,9 +33,12 @@ pipeline {
stages { stages {
stage ("gDashboard") { stage ("gDashboard") {
steps { steps {
script { script {
//retrieve MR data from gitlab and export to gSheet dir ("ci-scripts/ran_dashboard") {
sh returnStdout: true, script: 'python3 ci-scripts/ran_dashboard.py' //retrieve MR data from gitlab / mySQL db, build HTML pages and load them to AWS S3 bucket (configured as static web page hosting)
//deprecated method : sh returnStdout: true, script: 'python3 ran_dashboard.py'
sh returnStdout: true, script: 'python3 Hdashboard.py'
}
} }
} }
} }
......
version: '2'
services:
mysql:
container_name: oaicicd_mysql
restart: always
image: mysql:latest
environment:
MYSQL_ROOT_PASSWORD: 'ucZBc2XRYdvEm59F'
ports:
- "3307:3306"
volumes:
- /home/oaicicd/mysql/data:/var/lib/mysql
import pymysql
import sys
from datetime import datetime
#This is the script used to write the test results to the mysql DB
#Called by Jenkins pipeline (jenkinsfile)
#Must be located in /home/oaicicid/mysql on the database host
#Usage from Jenkinsfile :
#python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} ''
class SQLConnect:
def __init__(self):
self.connection = pymysql.connect(
host='172.22.0.2',
user='root',
password = 'ucZBc2XRYdvEm59F',
db='oaicicd_tests',
port=3306
)
def put(self,TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS):
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
cur=self.connection.cursor()
cur.execute ('INSERT INTO test_results (TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS,DATE) VALUES (%s,%s,%s,%s,%s,%s,%s);' , (TEST, MR, BRANCH, BUILD, BUILD_LINK, STATUS, now))
self.connection.commit()
self.connection.close()
if __name__ == "__main__":
mydb=SQLConnect()
mydb.put(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6])
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements. See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# * http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# * contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Merge Requests Dashboard for RAN on googleSheet
#
# Required Python Version
# Python 3.x
#
#---------------------------------------------------------------------
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
#Author Remi
import boto3
import shlex
import subprocess
import json #json structures
import datetime #now() and date formating
from datetime import datetime
import re
import gitlab
import yaml
import os
import time
from sqlconnect import SQLConnect
#-----------------------------------------------------------
# Class Declaration
#-----------------------------------------------------------
class Dashboard:
def __init__(self):
#init with data sources : git, yaml config file, test results databases
print("Collecting Data")
cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """
self.git = self.__getGitData(cmd) #git data from Gitlab
self.tests = self.__loadCfg('ran_dashboard_cfg.yaml') #tests table setup from yaml
self.db = self.__loadFromDB() #test results from database
def __loadCfg(self,yaml_file):
with open(yaml_file,'r') as f:
tests = yaml.load(f)
return tests
def __getGitData(self,cmd):
#cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """
process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
output = process.stdout.readline()
tmp=output.decode("utf-8")
d = json.loads(tmp)
return d
def __loadFromDB(self):
mr_list=[]
for x in range(len(self.git)):
mr_list.append(str(self.git[x]['iid']))
mydb=SQLConnect()
for MR in mr_list:
mydb.get(MR)
mydb.close_connection()
return mydb.data
def Test_initHTML(self, date):
self.f_html.write('<!DOCTYPE html>\n')
self.f_html.write('<head>\n')
self.f_html.write('<link rel="stylesheet" href="test_styles.css">\n')
self.f_html.write('<title>Test Dashboard</title>\n')
self.f_html.write('</head>\n')
self.f_html.write('<br>\n')
self.f_html.write('<br>\n')
self.f_html.write('<table>\n')
self.f_html.write('<tr>\n')
self.f_html.write('<td class="Main">OAI RAN TEST Status Dashboard</td>\n')
self.f_html.write('</td>\n')
self.f_html.write('<tr>\n')
self.f_html.write('<td class="DashLink"> <a href="https://oairandashboard.s3.eu-west-1.amazonaws.com/index.html">Merge Requests Dashboard</a></td>\n')
self.f_html.write('</td>\n')
self.f_html.write('<tr></tr>\n')
self.f_html.write('<tr>\n')
self.f_html.write('<td class="Date">Update : '+date+'</td>\n')
self.f_html.write('</td>\n')
self.f_html.write('</tr>\n')
self.f_html.write('</table>\n')
self.f_html.write('<br>\n')
self.f_html.write('<br>\n')
def Test_terminateHTML(self):
self.f_html.write('</body>\n')
self.f_html.write('</html>\n')
self.f_html.close()
def MR_initHTML(self,date):
self.f_html.write('<!DOCTYPE html>\n')
self.f_html.write('<head>\n')
self.f_html.write('<link rel="stylesheet" href="mr_styles.css">\n')
self.f_html.write('<title>MR Dashboard</title>\n')
self.f_html.write('</head>\n')
self.f_html.write('<br>\n')
self.f_html.write('<br>\n')
self.f_html.write('<table>\n')
self.f_html.write('<tr>\n')
self.f_html.write('<td class="Main">OAI RAN MR Status Dashboard</td>\n')
self.f_html.write('</td>\n')
self.f_html.write('<tr>\n')
self.f_html.write('<td class="DashLink"> <a href="https://oaitestdashboard.s3.eu-west-1.amazonaws.com/index.html">Tests Dashboard</a></td>\n')
self.f_html.write('</td>\n')
self.f_html.write('<tr></tr>\n')
self.f_html.write('<tr>\n')
self.f_html.write('<td class="Date">Update : '+date+'</td>\n')
self.f_html.write('</td>\n')
self.f_html.write('</tr>\n')
self.f_html.write('</table>\n')
self.f_html.write('<br>\n')
self.f_html.write('<br>\n')
self.f_html.write('<table class="MR_Table">\n')
self.f_html.write('<tr>\n')
self.f_html.write('<th class="MR">MR</th>\n')
self.f_html.write('<th class="CREATED_AT">Created_At</th>\n')
self.f_html.write('<th class="AUTHOR">Author</th>\n')
self.f_html.write('<th class="TITLE">Title</th>\n')
self.f_html.write('<th class="ASSIGNEE">Assignee</th>\n')
self.f_html.write('<th class="REVIEWER">Reviewer</th>\n')
self.f_html.write('<th class="CAN_START">CAN START</th>\n')
self.f_html.write('<th class="IN_PROGRESS">IN PROGRESS</th>\n')
self.f_html.write('<th class="COMPLETED">COMPLETED</th>\n')
self.f_html.write('<th class="REVIEW_FORM">Review Form</th>\n')
self.f_html.write('<th class="OK_MERGE">OK Merge</th>\n')
self.f_html.write('<th class="MERGE_CONFLICTS">Merge Conflicts</th>\n')
self.f_html.write('</tr>\n')
def MR_terminateHTML(self):
self.f_html.write('</table> \n')
self.f_html.write('</body>\n')
self.f_html.write('</html>\n')
self.f_html.close()
def MR_rowHTML(self,row):
self.f_html.write('<tr>\n')
self.f_html.write('<td><a href=\"'+row[0]+'\">'+row[1]+'</a></td>\n')
self.f_html.write('<td>'+row[2]+'</td>\n')
self.f_html.write('<td>'+row[3]+'</td>\n')
self.f_html.write('<td class="title_cell">'+row[4]+'</td>\n')
self.f_html.write('<td>'+row[5]+'</td>\n')
self.f_html.write('<td>'+row[6]+'</td>\n')
if row[7]=='X':
self.f_html.write('<td style="background-color: orange;">'+row[7]+'</td>\n')
else:
self.f_html.write('<td></td>\n')
if row[8]=='X':
self.f_html.write('<td style="background-color: yellow;">'+row[8]+'</td>\n')
else:
self.f_html.write('<td></td>\n')
if row[9]=='X':
self.f_html.write('<td style="background-color: rgb(144, 221, 231);">'+row[9]+'</td>\n')
else:
self.f_html.write('<td></td>\n')
if row[10]=='X':
self.f_html.write('<td style="background-color: rgb(58, 236, 58);">'+row[10]+'</td>\n')
else:
self.f_html.write('<td></td>\n')
if row[11]=='X':
self.f_html.write('<td style="background-color: rgb(58, 236, 58);">'+row[11]+'</td>\n')
else:
self.f_html.write('<td></td>\n')
if row[12]=='YES':
self.f_html.write('<td style="background-color: red;">'+row[12]+'</td>\n')
else:
self.f_html.write('<td></td>\n')
self.f_html.write('</tr>\n')
def Build(self, type, htmlfilename):
if type=='MR':
self.Build_MR_Table(htmlfilename)
elif type=='Tests':
self.Build_Test_Table(htmlfilename)
else :
print("Undefined Dashboard Type, options : MR or Tests")
def Build_Test_Table(self,htmlfilename):
print("Building Tests Dashboard...")
self.f_html=open(htmlfilename,'w')
###update date/time, format dd/mm/YY H:M:S
now = datetime.now()
dt_string = now.strftime("%d/%m/%Y %H:%M")
#HTML table header
self.Test_initHTML(dt_string)
#1 table per MR if test results exist
for x in range(len(self.git)):
mr=str(self.git[x]['iid'])
if 'PASS' not in self.db[mr]:
self.f_html.write('<h3><a href="https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/'+mr+'">'+mr+'</a>'+' '+self.git[x]['title'] + '</h3>\n')
self.f_html.write('<table class="Test_Table">\n')
self.f_html.write('<tr>\n')
self.f_html.write('<th class="Test_Name">Test Name</th>\n')
self.f_html.write('<th class="Test_Descr">Bench</th> \n')
self.f_html.write('<th class="Test_Descr">Test</th> \n')
self.f_html.write('<th class="Pass"># Pass</th>\n')
self.f_html.write('<th class="Fail"># Fail</th>\n')
self.f_html.write('<th class="Last_Pass">Last Pass</th>\n')
self.f_html.write('<th class="Last_Fail">Last Fail</th>\n')
self.f_html.write('</tr>\n')
#parsing the tests
for t in self.tests:
row=[]
short_name= t
hyperlink= self.tests[t]['link']
job=self.tests[t]['job']
if job in self.db[mr]:
if 'PASS' in self.db[mr][job]:
row.append(self.db[mr][job]['PASS'])
else:
row.append('')
if 'FAIL' in self.db[mr][job]:
row.append(self.db[mr][job]['FAIL'])
else:
row.append('')
#2 columns for last_pass and last_fail links
if 'last_pass' in self.db[mr][job]:
lastpasshyperlink= self.db[mr][job]['last_pass'][1]
lastpasstext= self.db[mr][job]['last_pass'][0]
else:
lastpasshyperlink=''
lastpasstext=''
if 'last_fail' in self.db[mr][job]:
lastfailhyperlink= self.db[mr][job]['last_fail'][1]
lastfailtext= self.db[mr][job]['last_fail'][0]
else:
lastfailhyperlink=''
lastfailtext=''
self.f_html.write('<tr>\n')
self.f_html.write('<td><a href='+hyperlink+'>'+short_name+'</a></td>\n')
self.f_html.write('<td>'+self.tests[t]['bench']+'</td>\n')
self.f_html.write('<td>'+self.tests[t]['test']+'</td>\n')
if row[0]!='':
self.f_html.write('<td style="background-color: rgb(58, 236, 58);">'+str(row[0])+'</td>\n')
else:
self.f_html.write('<td></td>\n')
if row[1]!='':
self.f_html.write('<td style="background-color: red;">'+str(row[1])+'</td>\n')
else:
self.f_html.write('<td></td>\n')
self.f_html.write('<td><a href='+lastpasshyperlink+'>'+lastpasstext+'</a></td>\n')
self.f_html.write('<td><a href='+lastfailhyperlink+'>'+lastfailtext+'</a></td>\n')
self.f_html.write('</tr>\n')
self.f_html.write('</table>\n')
#terminate HTML table and close file
self.Test_terminateHTML()
def Build_MR_Table(self,htmlfilename):
print("Building Merge Requests Dashboard...")
self.f_html=open(htmlfilename,'w')
###update date/time, format dd/mm/YY H:M:S
now = datetime.now()
dt_string = now.strftime("%d/%m/%Y %H:%M")
#HTML table header
self.MR_initHTML(dt_string)
###MR data lines
for x in range(len(self.git)):
hyperlink= 'https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/'+ str(self.git[x]['iid'])
text= str(self.git[x]['iid'])
date_time_str = self.git[x]['created_at']
date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ')
milestone1=milestone2=milestone3=milestone4=""
if self.git[x]['milestone']!=None:
if self.git[x]['milestone']['title']=="REVIEW_CAN_START":
milestone1="X"
elif self.git[x]['milestone']['title']=="REVIEW_IN_PROGRESS":
milestone2="X"
elif self.git[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED":
milestone3="X"
elif self.git[x]['milestone']['title']=="OK_TO_BE_MERGED":
milestone4="X"
else:
pass
else:
pass
#check if empty or not
if self.git[x]['assignee']!=None:
assignee = str(self.git[x]['assignee']['name'])
else:
assignee = ""
#check if empty or not
if len(self.git[x]['reviewers'])!=0:
reviewer = str(self.git[x]['reviewers'][0]['name'])
else:
reviewer = ""
if self.git[x]['has_conflicts']==True:
conflicts = "YES"
else:
conflicts = ""
#add a column flagging that the review form is present
#we use gitlab API to parse the MR notes
gl = gitlab.Gitlab.from_config('OAI')
project_id = 223
project = gl.projects.get(project_id)
#get the opened MR in the project
mrs = project.mergerequests.list(state='opened',per_page=100)
review_form=''
for m in range (0,len(mrs)):
if mrs[m].iid==self.git[x]['iid']:#check the iid is the one we are on
mr_notes = mrs[m].notes.list(all=True)
n=0
found=False
while found==False and n<len(mr_notes):
res=re.search('Code Review by',mr_notes[n].body)#this is the marker we are looking for in all notes
if res!=None:
review_form = "X"
found=True
n+=1
#build final row to be inserted
row =[hyperlink, text, str(date_time_obj.date()),str(self.git[x]['author']['name']), str(self.git[x]['title']),\
assignee, reviewer,\
milestone1,milestone2,milestone3,review_form,milestone4,conflicts]
self.MR_rowHTML(row)
#terminate HTML table and close file
self.MR_terminateHTML()
def CopyToS3(self,htmlfilename,bucket,key):
print("Uploading to S3 bucket")
#Creating Session With Boto3.
s3 = boto3.client('s3')
#Creating S3 Resource From the Session.
result = s3.upload_file(htmlfilename, bucket,key, ExtraArgs={'ACL':'public-read','ContentType': 'text/html'})
def main():
htmlDash=Dashboard()
htmlDash.Build('MR','/tmp/MR_index.html')
htmlDash.CopyToS3('/tmp/MR_index.html','oairandashboard','index.html')
htmlDash.Build('Tests','/tmp/Tests_index.html')
htmlDash.CopyToS3('/tmp/Tests_index.html','oaitestdashboard','index.html')
if __name__ == "__main__":
# execute only if run as a script
main()
body {
font-family: 'lato', sans-serif;
}
.Main {
text-align:left;
font-size: 30px;
font-weight: bold;
}
.DashLink {
text-align:left;
font-size: 16px;
}
.DashLink:hover {
font-weight: bold;
}
.Date {
text-align:left;
font-size: 16px;
font-weight: bold;
}
a { text-decoration: none; }
a:visited { text-decoration: none; color:blue}
a:hover { text-decoration: none; font-weight: bold; color:blue}
.MR_Table {
border-collapse: collapse;
}
.MR_Table tr {
font-size: 12px;
text-align:center;
}
.MR_Table tr:hover {
background-color:lightgray;
}
.MR_Table td {
border: 1px solid black;
padding : 5px;
}
.MR_Table td:hover {
font-weight : bold;
}
.MR_Table th {
font-size: 14px;
text-align:center;
padding : 5px;
}
.title_cell {
text-align:left;
}
.MR {
table-layout: fixed;
width: 50px;
background-color: rgb(143, 154, 216);
}
.CREATED_AT {
table-layout: fixed;
width: 100px;
background-color: rgb(143, 154, 216);
}
.AUTHOR {
table-layout: fixed;
width: 150px;
background-color: rgb(143, 154, 216);
}
.TITLE {
table-layout: fixed;
width: 500px;
background-color: rgb(143, 154, 216);
}
.ASSIGNEE {
table-layout: fixed;
width: 150px;
background-color: rgb(143, 154, 216);
}
.REVIEWER {
table-layout: fixed;
width: 150px;
background-color: rgb(143, 154, 216);
}
.CAN_START {
background-color: orange;
table-layout: fixed;
width: 150px;
}
.IN_PROGRESS {
background-color: yellow;
table-layout: fixed;
width: 150px;
}
.COMPLETED {
background-color: rgb(144, 221, 231);
table-layout: fixed;
width: 150px;
}
.REVIEW_FORM {
table-layout: fixed;
width: 100px;
background-color: rgb(143, 154, 216);
}
.OK_MERGE {
background-color: rgb(58, 236, 58);
table-layout: fixed;
width: 150px;
}
.MERGE_CONFLICTS {
table-layout: fixed;
width: 100px;
background-color: rgb(143, 154, 216);
}
...@@ -30,6 +30,8 @@ ...@@ -30,6 +30,8 @@
# Import # Import
#----------------------------------------------------------- #-----------------------------------------------------------
#author Remi
#import google spreadsheet API #import google spreadsheet API
import gspread import gspread
from oauth2client.service_account import ServiceAccountCredentials from oauth2client.service_account import ServiceAccountCredentials
...@@ -42,6 +44,13 @@ import datetime #now() and date formating ...@@ -42,6 +44,13 @@ import datetime #now() and date formating
from datetime import datetime from datetime import datetime
import re import re
import gitlab import gitlab
import yaml
import os
import pickle
import time
from sqlconnect import SQLConnect
#----------------------------------------------------------- #-----------------------------------------------------------
# Class Declaration # Class Declaration
...@@ -57,48 +66,78 @@ class gDashboard: ...@@ -57,48 +66,78 @@ class gDashboard:
#worksheet #worksheet
self.sheet = self.ss.worksheet(worksheet) self.sheet = self.ss.worksheet(worksheet)
self.ss.del_worksheet(self.sheet) #start by deleting the old sheet self.ss.del_worksheet(self.sheet) #start by deleting the old sheet
self.sheet = self.ss.add_worksheet(title=worksheet, rows="100", cols="20") #create a new one self.sheet = self.ss.add_worksheet(title=worksheet, rows="100", cols="30") #create a new one
self.d = {} #data dictionary
#init with data sources : git, yaml config file, test results databases
cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """
self.git = self.__getGitData(cmd) #git data from Gitlab
self.tests = self.__loadCfg('ran_dashboard_cfg.yaml') #tests table setup from yaml
self.db = self.__loadFromDB() #test results from database
def fetchData(self,cmd):
def __loadCfg(self,yaml_file):
with open(yaml_file,'r') as f:
tests = yaml.load(f)
return tests
def __getGitData(self,cmd):
#cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ #cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """
process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
output = process.stdout.readline() output = process.stdout.readline()
tmp=output.decode("utf-8") tmp=output.decode("utf-8")
self.d = json.loads(tmp) d = json.loads(tmp)
return d
def __loadFromDB(self):
mr_list=[]
for x in range(len(self.git)):
mr_list.append(str(self.git[x]['iid']))
mydb=SQLConnect()
for MR in mr_list:
mydb.get(MR)
mydb.close_connection()
return mydb.data
def gBuild(self, destinationSheetName): def gBuild(self, destinationSheetName):
#line 1 : update date/time, format dd/mm/YY H:M:S ###line 1 : update date/time, format dd/mm/YY H:M:S
now = datetime.now() now = datetime.now()
dt_string = "Update : " + now.strftime("%d/%m/%Y %H:%M") dt_string = "Update : " + now.strftime("%d/%m/%Y %H:%M")
row =[dt_string] row =[dt_string]
self.sheet.insert_row(row, index=1, value_input_option='RAW') self.sheet.insert_row(row, index=1, value_input_option='RAW')
#line 2 empty ###line 2 is for the test short names (links to jenkins pipeline), updated at the end
#line 3 is for the column names
###line 3 is for the column names
i=3 i=3
row =["MR","Created_at","Author","Title","Assignee", "Reviewer", "CAN START","IN PROGRESS","COMPLETED","Review Form","OK MERGE","Merge conflicts"] row =["MR","Created_at","Author","Title","Assignee", "Reviewer", "CAN START","IN PROGRESS","COMPLETED","Review Form","OK MERGE","Merge conflicts",""]
#tests
for t in range(0,len(self.tests)):
row.append("# PASS")
row.append("# FAIL")
row.append("Last Pass")
row.append("Last Fail")
self.sheet.insert_row(row, index=i, value_input_option='RAW') self.sheet.insert_row(row, index=i, value_input_option='RAW')
#line 4 onward, MR data lines ###line 4 onward, MR data lines
for x in range(len(self.d)): for x in range(len(self.git)):
i=i+1 i=i+1
date_time_str = self.d[x]['created_at'] date_time_str = self.git[x]['created_at']
date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ') date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ')
milestone1=milestone2=milestone3=milestone4="" milestone1=milestone2=milestone3=milestone4=""
if self.d[x]['milestone']!=None: if self.git[x]['milestone']!=None:
if self.d[x]['milestone']['title']=="REVIEW_CAN_START": if self.git[x]['milestone']['title']=="REVIEW_CAN_START":
milestone1="X" milestone1="X"
elif self.d[x]['milestone']['title']=="REVIEW_IN_PROGRESS": elif self.git[x]['milestone']['title']=="REVIEW_IN_PROGRESS":
milestone2="X" milestone2="X"
elif self.d[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED": elif self.git[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED":
milestone3="X" milestone3="X"
elif self.d[x]['milestone']['title']=="OK_TO_BE_MERGED": elif self.git[x]['milestone']['title']=="OK_TO_BE_MERGED":
milestone4="X" milestone4="X"
else: else:
pass pass
...@@ -106,18 +145,18 @@ class gDashboard: ...@@ -106,18 +145,18 @@ class gDashboard:
pass pass
#check if empty or not #check if empty or not
if self.d[x]['assignee']!=None: if self.git[x]['assignee']!=None:
assignee = str(self.d[x]['assignee']['name']) assignee = str(self.git[x]['assignee']['name'])
else: else:
assignee = "" assignee = ""
#check if empty or not #check if empty or not
if len(self.d[x]['reviewers'])!=0: if len(self.git[x]['reviewers'])!=0:
reviewer = str(self.d[x]['reviewers'][0]['name']) reviewer = str(self.git[x]['reviewers'][0]['name'])
else: else:
reviewer = "" reviewer = ""
if self.d[x]['has_conflicts']==True: if self.git[x]['has_conflicts']==True:
conflicts = "YES" conflicts = "YES"
else: else:
conflicts = "" conflicts = ""
...@@ -129,13 +168,13 @@ class gDashboard: ...@@ -129,13 +168,13 @@ class gDashboard:
project_id = 223 project_id = 223
project = gl.projects.get(project_id) project = gl.projects.get(project_id)
#get the opened MR in the project #get the opened MR in the project
mrs = project.mergerequests.list(state='opened') mrs = project.mergerequests.list(state='opened',per_page=100)
review_form=''
for m in range (0,len(mrs)): for m in range (0,len(mrs)):
if mrs[m].iid==self.d[x]['iid']:#check the iid is the one we are on if mrs[m].iid==self.git[x]['iid']:#check the iid is the one we are on
mr_notes = mrs[m].notes.list(all=True) mr_notes = mrs[m].notes.list(all=True)
n=0 n=0
found=False found=False
review_form=""
while found==False and n<len(mr_notes): while found==False and n<len(mr_notes):
res=re.search('Code Review by',mr_notes[n].body)#this is the marker we are looking for in all notes res=re.search('Code Review by',mr_notes[n].body)#this is the marker we are looking for in all notes
if res!=None: if res!=None:
...@@ -143,29 +182,81 @@ class gDashboard: ...@@ -143,29 +182,81 @@ class gDashboard:
found=True found=True
n+=1 n+=1
#build final row to be inserted, the first column is left empty for now, will be filled afterward with hyperlinks to gitlab MR #build final row to be inserted, the first column is left empty for now, will be filled afterward with hyperlinks to gitlab MR
row =["", str(date_time_obj.date()),str(self.d[x]['author']['name']),str(self.d[x]['title']),\ row =["", str(date_time_obj.date()),str(self.git[x]['author']['name']), str(self.git[x]['title']),\
assignee, reviewer,\ assignee, reviewer,\
milestone1,milestone2,milestone3,review_form,milestone4,conflicts] milestone1,milestone2,milestone3,review_form,milestone4,conflicts,""]
#and append the test results coming from self.db
#insert the row to worksheet mr=str(self.git[x]['iid'])
for t in self.tests:
if mr in self.db:
job=self.tests[t]['job']
if job in self.db[mr]:
if 'PASS' in self.db[mr][job]:
row.append(self.db[mr][job]['PASS'])
else:
row.append('')
if 'FAIL' in self.db[mr][job]:
row.append(self.db[mr][job]['FAIL'])
else:
row.append('')
#leave 2 columns for last_pass and last_fail links
row.append('')
row.append('')
else:
#4 columns are empty
row.append('')
row.append('')
row.append('')
row.append('')
#insert the final row to worksheet
self.sheet.insert_row(row, index=i, value_input_option='RAW') self.sheet.insert_row(row, index=i, value_input_option='RAW')
time.sleep(10)
#add MR hyperlinks in a list of requests to be sent as one update batch; this to save API calls (quotas) #add MR hyperlinks in a list of requests to be sent as one update batch; this to save API calls (quotas)
i=3 i=3
requests=[] requests=[]
for x in range(len(self.d)): for x in range(len(self.git)):
rowIndex=i rowIndex=i
colIndex=0 colIndex=0
hyperlink= '\"'+"https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/"+ str(self.d[x]['iid']) +'\"' hyperlink= '\"'+"https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/"+ str(self.git[x]['iid']) +'\"'
text= '\"'+str(self.d[x]['iid'])+'"' text= '\"'+str(self.git[x]['iid'])+'"'
requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex)) requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex))
i=i+1
mr=str(self.git[x]['iid'])
colIndex=15
for t in self.tests:
job=self.tests[t]['job']
if job in self.db[mr]:
if 'last_pass' in self.db[mr][job]:
hyperlink= '\"'+ self.db[mr][job]['last_pass'][1] +'\"'
text= '\"'+self.db[mr][job]['last_pass'][0]+'"'
requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex))
if 'last_fail' in self.db[mr][job]:
hyperlink= '\"'+ self.db[mr][job]['last_fail'][1] +'\"'
text= '\"'+self.db[mr][job]['last_fail'][0]+'"'
requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex+1))
colIndex+=4 #move to next test
i=i+1 #increment row index for next MR
body = {"requests": requests} body = {"requests": requests}
self.ss.batch_update(body) self.ss.batch_update(body)
###line 2 is for the test names
#add MR hyperlinks in a list of requests to be sent as one update batch; this to save API calls (quotas)
requests=[]
rowIndex=1
colIndex=13
for t in self.tests :
hyperlink= '\"'+self.tests[t]['link']+'\"'
short_name= '\"'+ t +'\"'
requests.append(self.addHyperlink(hyperlink, short_name, destinationSheetName, rowIndex, colIndex))
colIndex+=4
body = {"requests": requests}
self.ss.batch_update(body)
def addHyperlink(self, hyperlink, text, destinationSheetName, rowIndex, colIndex): def addHyperlink(self, hyperlink, text, destinationSheetName, rowIndex, colIndex):
...@@ -207,17 +298,17 @@ class gDashboard: ...@@ -207,17 +298,17 @@ class gDashboard:
"copyPaste": { "copyPaste": {
"source": { "source": {
"sheetId": sourceSheetId, "sheetId": sourceSheetId,
"startRowIndex": 0, "startRowIndex": 1,
"endRowIndex": 40, "endRowIndex": 40,
"startColumnIndex": 0, "startColumnIndex": 0,
"endColumnIndex": 12 "endColumnIndex": 30
}, },
"destination": { "destination": {
"sheetId": destinationSheetId, "sheetId": destinationSheetId,
"startRowIndex": 0, "startRowIndex": 1,
"endRowIndex": 40, "endRowIndex": 40,
"startColumnIndex": 0, "startColumnIndex": 0,
"endColumnIndex": 12 "endColumnIndex": 30
}, },
"pasteType": "PASTE_FORMAT" "pasteType": "PASTE_FORMAT"
} }
...@@ -280,15 +371,30 @@ class gDashboard: ...@@ -280,15 +371,30 @@ class gDashboard:
} }
) )
body = {"requests": requests}
self.ss.batch_update(body)
#group MR related columns
# sheetId = self.ss.worksheet(destinationSheetName)._properties['sheetId']
# requests.append(
# {
# "addDimensionGroup": {
# "range": {
# "dimension": "COLUMNS",
# "sheetId": sheetId,
# "startIndex": 3,
# "endIndex": 12
# },
# }
# }
# )
#
body = {"requests": requests}
self.ss.batch_update(body)
def main(): def main():
my_gDashboard=gDashboard("/opt/dashboard/g_creds.json", 'OAI RAN Dashboard', 'MR Status') my_gDashboard=gDashboard("/opt/dashboard/g_creds.json", 'OAI RAN Dashboard', 'MR Status')
cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """
my_gDashboard.fetchData(cmd)
my_gDashboard.gBuild("MR Status") my_gDashboard.gBuild("MR Status")
my_gDashboard.gFormat("Formating" , "MR Status") my_gDashboard.gFormat("Formating" , "MR Status")
......
LTE-2x2 : #short name used in the dashboard
job : 'RAN-LTE-2x2-Module-OAIEPC' #job name from Jenkins, used in the database
link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-LTE-2x2-Module-OAIEPC'
bench : 'Obelix-N310-OAIEPC-Quectel(nrmodule2)'
test : 'TM2, TDD, 40MHz, MCS9, 26Mb DL, 7Mb UL'
NSA-B200 :
job : 'RAN-NSA-B200-Module-LTEBOX'
link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-NSA-B200-Module-LTEBOX'
bench : 'Nepes-B200-Obelix-B200-LTEBOX-Quectel(idefix)'
test : 'TM1, 20MHz, 60Mb DL, 3Mb UL'
NSA-2x2 :
job : 'RAN-NSA-2x2-Module-OAIEPC'
link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-NSA-2x2-Module-OAIEPC'
bench : 'Asterix-N310-Obelix-N310-OAIEPC-Quectel(nrmodule2)'
test : 'TM2, TDD, 40MHz, 60Mb DL, 3Mb UL'
SA-N310 :
job : 'RAN-SA-Module-CN5G'
link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-SA-Module-CN5G'
bench : 'Asterix-N310-OAICN5G-Quectel(nrmodule2)'
test : 'TM2, TDD, 40MHz, 60Mb DL, 3Mb UL'
The code ran_dashboard.py was initially developped to bring MR status and test resuts to a google sheet, using google sheets API.
This method is now deprecated, and replaced by Hdashboard.py that builds the results as HTML page and loads it to AWS S3.
ran_dashboard_cfg.yaml is still used by Hdashboard.py as tests config file.
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements. See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# * http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# * contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Merge Requests Dashboard for RAN on googleSheet
#
# Required Python Version
# Python 3.x
#
#---------------------------------------------------------------------
#author Remi
import pymysql
import sys
from datetime import datetime
import pickle
#This is the script/package used by the dashboard to retrieve the MR test results from the database
class SQLConnect:
def __init__(self):
self.connection = pymysql.connect(
host='172.22.0.2',
user='root',
password = 'ucZBc2XRYdvEm59F',
db='oaicicd_tests',
port=3306
)
self.data={}
#retrieve data from mysql database and organize it in a dictionary (per MR passed as argument)
def get(self,MR):
self.data[MR]={}
cur=self.connection.cursor()
#get counters per test
sql = "select TEST,STATUS, count(*) AS COUNT from test_results where MR=(%s) group by TEST, STATUS;"
cur.execute(sql,MR)
response=cur.fetchall()
if len(response)==0:#no test results yet
self.data[MR]['PASS']=''
self.data[MR]['FAIL']=''
else:
for i in range(0,len(response)):
test=response[i][0]
status=response[i][1]
count=response[i][2]
if test in self.data[MR]:
self.data[MR][test][status]=count
else:
self.data[MR][test]={}
self.data[MR][test][status]=count
#get last failing build and link
sql = "select TEST,BUILD, BUILD_LINK from test_results where MR=(%s) and STATUS='FAIL' order by DATE DESC;"
cur.execute(sql,MR)
response=cur.fetchall()
if len(response)!=0:
for i in range(0,len(response)):
test=response[i][0]
build=response[i][1]
link=response[i][2]
if 'last_fail' not in self.data[MR][test]:
self.data[MR][test]['last_fail']=[]
self.data[MR][test]['last_fail'].append(build)
self.data[MR][test]['last_fail'].append(link)
#get last passing build and link
sql = "select TEST,BUILD, BUILD_LINK from test_results where MR=(%s) and STATUS='PASS' order by DATE DESC;"
cur.execute(sql,MR)
response=cur.fetchall()
if len(response)!=0:
for i in range(0,len(response)):
test=response[i][0]
build=response[i][1]
link=response[i][2]
if 'last_pass' not in self.data[MR][test]:
self.data[MR][test]['last_pass']=[]
self.data[MR][test]['last_pass'].append(build)
self.data[MR][test]['last_pass'].append(link)
#close database connection
def close_connection(self):
self.connection.close()
body {
font-family: 'lato', sans-serif;
}
.Main {
text-align: left;
font-size: 30px;
font-weight: bold;
}
.DashLink {
text-align:left;
font-size: 16px;
}
.DashLink:hover {
font-weight: bold;
}
.Date {
text-align: left;
font-size: 16px;
font-weight: bold;
}
h3 {
font-size: 16px;
}
a { text-decoration: none; }
a:visited { text-decoration: none; color:blue}
a:hover { text-decoration: none; font-weight: bold; color:blue}
.Test_Table {
border-collapse: collapse;
}
.Test_Table tr {
font-size: 12px;
text-align:center;
}
.Test_Table tr:hover {
background-color:lightgray;
}
.Test_Table td {
border: 1px solid black;
padding : 5px;
}
.Test_Table td:hover {
font-weight : bold;
}
.Test_Table th {
font-size: 14px;
text-align:center;
padding : 5px;
}
.Test_Name {
table-layout: fixed;
width: 100px;
background-color: rgb(143, 154, 216);
}
.Test_Descr {
table-layout: fixed;
width: 400px;
background-color: rgb(143, 154, 216);
}
.Pass {
table-layout: fixed;
width: 100px;
background-color: rgb(143, 154, 216);
}
.Fail {
table-layout: fixed;
width: 100px;
background-color: rgb(143, 154, 216);
}
.Last_Pass {
table-layout: fixed;
width: 100px;
background-color: rgb(143, 154, 216);
}
.Last_Fail {
table-layout: fixed;
width: 100px;
background-color: rgb(143, 154, 216);
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment