diff --git a/ci-scripts/Jenkinsfile-GitLab-Docker b/ci-scripts/Jenkinsfile-GitLab-Docker index a6dd667143e40d335956fd38b36bd3b902106815..00b92c8cb9524262156765b75cd212399a3df995 100644 --- a/ci-scripts/Jenkinsfile-GitLab-Docker +++ b/ci-scripts/Jenkinsfile-GitLab-Docker @@ -236,17 +236,21 @@ pipeline { sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Deploy --service vpp-upf --upf_tag ' + upf_tag // SMF is dependant on AMF and EXT-DN: they will be deploy as well sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Deploy --service oai-smf' + // GNB SIM as RAN emulator + sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Deploy --docker_compose docker-compose-gnbsim.yaml --service gnbsim-vpp' // Check if deployment is OK sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Check --service all' // Undeploy once OK + sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --docker_compose docker-compose-gnbsim.yaml --service gnbsim-vpp' sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --service all' } } post { unsuccessful { - sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --service all' + sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode Check --service all || true' + sh 'python3 ./ci-scripts/sanityCheckDeployment.py --mode UnDeploy --service all || true' } } } diff --git a/ci-scripts/generateHtmlReport.py b/ci-scripts/generateHtmlReport.py index 05cb433faba0b5379d2675f6304b19937a3e9b60..7a1b2923240d0ac9a5c2a2757f50174430069ecc 100644 --- a/ci-scripts/generateHtmlReport.py +++ b/ci-scripts/generateHtmlReport.py @@ -656,10 +656,92 @@ class HtmlReport(): def testSummaryHeader(self): self.file.write(' <h2>Test Summary</h2>\n') - self.file.write(' <div class="alert alert-warning">\n') - self.file.write(' <strong>Not performed yet. <span class="glyphicon glyphicon-warning-sign"></span></strong>\n') + cwd = os.getcwd() + if os.path.isfile(cwd + '/archives/deployment_status.log'): + cmd = 'egrep -c "DEPLOYMENT: OK" archives/deployment_status.log || true' + status = False + ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, encoding='utf-8') + if ret.stdout is not None: + status = True + if status: + self.file.write(' <div class="alert alert-success">\n') + self.file.write(' <strong>Successful Sanity Check Deployment! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n') + self.file.write(' </div>\n') + else: + self.file.write(' <div class="alert alert-danger">\n') + self.file.write(' <strong>Failed Sanity Check Deployment! <span class="glyphicon glyphicon-warning-sign"></span></strong>\n') + self.file.write(' </div>\n') + else: + self.file.write(' <div class="alert alert-warning">\n') + self.file.write(' <strong>Not performed. <span class="glyphicon glyphicon-warning-sign"></span></strong>\n') + self.file.write(' </div>\n') + self.file.write(' <br>\n') + self.file.write(' <button data-toggle="collapse" data-target="#deployment-details">More details on Deployment</button>\n') + self.file.write(' <br>\n') + self.file.write(' <div id="deployment-details" class="collapse">\n') + self.file.write(' <br>\n') + self.file.write(' <table class="table-bordered" width = "80%" align = "center" border = 1>\n') + self.file.write(' <tr bgcolor = "#33CCFF" >\n') + self.file.write(' <th>Container Name</th>\n') + self.file.write(' <th>Used Image Tag</th>\n') + self.file.write(' <th>Image Creation Date</th>\n') + self.file.write(' <th>Used Image Size</th>\n') + self.file.write(' </tr>\n') + self.addImageRow('mysql') + self.addImageRow('oai_nrf') + self.addImageRow('oai_amf') + self.addImageRow('oai_smf') + self.addImageRow('oai_upf_vpp') + self.file.write(' </table>\n') self.file.write(' </div>\n') + def addImageRow(self, imageInfoPrefix): + cwd = os.getcwd() + if imageInfoPrefix == 'oai_amf': + containerName = 'oai-amf' + tagPattern = 'OAI_AMF_TAG' + if imageInfoPrefix == 'oai_smf': + containerName = 'oai-smf' + tagPattern = 'OAI_SMF_TAG' + if imageInfoPrefix == 'oai_nrf': + containerName = 'oai-nrf' + tagPattern = 'OAI_NRF_TAG' + if imageInfoPrefix == 'oai_upf_vpp': + containerName = 'vpp-upf' + tagPattern = 'OAI_UPF_VPP_TAG' + if imageInfoPrefix == 'mysql': + containerName = imageInfoPrefix + tagPattern = 'MYSQL_TAG' + if os.path.isfile(cwd + '/archives/' + imageInfoPrefix + '_image_info.log'): + usedTag = '' + createDate = '' + size = '' + with open(cwd + '/archives/' + imageInfoPrefix + '_image_info.log') as imageLog: + for line in imageLog: + line = line.strip() + result = re.search(tagPattern + ': (?P<tag>[a-zA-Z0-9\.\-\_:]+)', line) + if result is not None: + usedTag = result.group('tag') + result = re.search('Date = (?P<date>[a-zA-Z0-9\-\_:]+)', line) + if result is not None: + createDate = result.group('date') + result = re.search('Size = (?P<size>[0-9]+) bytes', line) + if result is not None: + sizeInt = int(result.group('size')) + if sizeInt < 1000000: + sizeInt = int(sizeInt / 1000) + size = str(sizeInt) + ' kB' + else: + sizeInt = int(sizeInt / 1000000) + size = str(sizeInt) + ' MB' + imageLog.close() + self.file.write(' <tr>\n') + self.file.write(' <td>' + containerName + '</td>\n') + self.file.write(' <td>' + usedTag + '</td>\n') + self.file.write(' <td>' + createDate + '</td>\n') + self.file.write(' <td>' + size + '</td>\n') + self.file.write(' </tr>\n') + def testSummaryFooter(self): self.file.write(' <br>\n') diff --git a/ci-scripts/sanityCheckDeployment.py b/ci-scripts/sanityCheckDeployment.py index 19ba6baa6b0dc300af920493b8ec031c1a3f4228..a40100b51a6af7c6d76e7e5ae38d66b02e18663d 100644 --- a/ci-scripts/sanityCheckDeployment.py +++ b/ci-scripts/sanityCheckDeployment.py @@ -23,8 +23,10 @@ import argparse import logging import os import re -import sys +import shutil import subprocess +import sys +import time logging.basicConfig( level=logging.DEBUG, @@ -36,12 +38,13 @@ def main() -> None: args = _parse_args() if args.mode == 'Deploy': - deployObject(args.service, args.upf_tag) + deployObject(args.service, args.upf_tag, args.docker_compose) if args.mode == 'Check': - logging.info ('Performing Check') + checkDeployment(args.service, args.docker_compose) + if args.mode == 'UnDeploy': - logging.info ('Un-Deploying ' + args.service) + undeployObject(args.service, args.docker_compose) def _parse_args() -> argparse.Namespace: """Parse the command line args @@ -82,10 +85,103 @@ def _parse_args() -> argparse.Namespace: ) return parser.parse_args() -def deployObject(service, tag): +def deployObject(service, tag, compose_file): logging.info ('Deploying ' + service) + + if service == 'gnbsim-vpp': + logging.info ('Not available for the moment') + return + + if not os.path.isdir('./archives'): + subprocess_run_w_echo('mkdir -p ./archives') + + os.chdir('docker-compose') + if not os.path.exists('./ci-' + compose_file): + logging.debug('cp ./' + compose_file + ' ./ci-' + compose_file) + shutil.copyfile('./' + compose_file, './ci-' + compose_file) + expect_healthy = 0 + nb_healthy = 0 + cnt = 0 + # mysql does not require image tag update + # Also starting a capture on the public docker network + if service == 'mysql': + subprocess_run_w_echo('echo "MYSQL_TAG: mysql:5.7" > ../archives/mysql_image_info.log') + subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" mysql:5.7 >> ../archives/mysql_image_info.log') + subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" mysql:5.7 >> ../archives/mysql_image_info.log') + subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service) + expect_healthy = 1 + subprocess_run_w_echo('sudo nohup tshark -i cn5g-public -f "not arp and not port 53 and not host archive.ubuntu.com and not host security.ubuntu.com" -w /tmp/ci-upf-vpp-sanity.pcap > /dev/null 2>&1 &') + # We might need tag manipulations + if service == 'oai-nrf': + subprocess_run_w_echo('echo "OAI_NRF_TAG: oai-nrf:develop" > ../archives/oai_nrf_image_info.log') + subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-nrf:develop >> ../archives/oai_nrf_image_info.log') + subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-nrf:develop >> ../archives/oai_nrf_image_info.log') + subprocess_run_w_echo('sed -i -e "s#oai-nrf:latest#oai-nrf:develop#" ./ci-' + compose_file) + subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service) + expect_healthy = 2 + if service == 'vpp-upf': + subprocess_run_w_echo('echo "OAI_UPF_VPP_TAG: oai-upf-vpp:' + tag + '" > ../archives/oai_upf_vpp_image_info.log') + subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-upf-vpp:' + tag + ' >> ../archives/oai_upf_vpp_image_info.log') + subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-upf-vpp:' + tag + ' >> ../archives/oai_upf_vpp_image_info.log') + subprocess_run_w_echo('sed -i -e "s#oai-upf-vpp:latest#oai-upf-vpp:' + tag + '#" ./ci-' + compose_file) + subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service) + expect_healthy = 3 + if service == 'oai-smf': + subprocess_run_w_echo('echo "OAI_AMF_TAG: oai-amf:develop" > ../archives/oai_amf_image_info.log') + subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-amf:develop >> ../archives/oai_amf_image_info.log') + subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-amf:develop >> ../archives/oai_amf_image_info.log') + subprocess_run_w_echo('echo "OAI_SMF_TAG: oai-smf:develop" > ../archives/oai_smf_image_info.log') + subprocess_run_w_echo('docker image inspect --format="Size = {{.Size}} bytes" oai-smf:develop >> ../archives/oai_smf_image_info.log') + subprocess_run_w_echo('docker image inspect --format="Date = {{.Created}}" oai-smf:develop >> ../archives/oai_smf_image_info.log') + subprocess_run_w_echo('sed -i -e "s#oai-amf:latest#oai-amf:develop#" -e "s#oai-smf:latest#oai-smf:develop#" ./ci-' + compose_file) + subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' up -d ' + service) + expect_healthy = 5 + + # Counting the number of healthy containers + cmd = 'docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' ps -a' + while (nb_healthy != expect_healthy) or (cnt > 50): + time.sleep(5) + cnt += 1 + res = subprocess.check_output(cmd, shell=True, universal_newlines=True) + nb_healthy = 0 + for line in res.split('\n'): + result = re.search('Up.*healthy', line) + if result is not None: + nb_healthy += 1 + if nb_healthy == expect_healthy: + logging.info('OK') + elif cnt > 50: + logging.error('KO') + sys.exit(-1) + +def undeployObject(service, compose_file): + logging.info ('Un-Deploying ' + service) + + if service == 'gnbsim-vpp': + logging.info ('Not available for the moment') + return + + os.chdir('docker-compose') + subprocess_run_w_echo('docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' down') + +def checkDeployment(service, compose_file): + logging.info ('Performing Check') + subprocess_run_w_echo('sudo chmod 666 /tmp/ci-upf-vpp-sanity.pcap') + if not os.path.isdir('./archives'): + subprocess_run_w_echo('mkdir -p ./archives') + + if os.path.exists('/tmp/ci-upf-vpp-sanity.pcap'): + subprocess_run_w_echo('cp /tmp/ci-upf-vpp-sanity.pcap archives') + os.chdir('docker-compose') - subprocess_run_w_echo ('ls') + cmd = 'docker-compose -p vpp-sanity -f ./ci-' + compose_file + ' ps -a' + res = subprocess.check_output(cmd, shell=True, universal_newlines=True) + for line in res.split('\n'): + result = re.search('Up.*healthy', line) + if result is not None: + subprocess_run_w_echo('docker logs ' + line.split(' ')[0] + ' > ../archives/' + line.split(' ')[0] + '.log 2>&1') + + subprocess_run_w_echo('echo "SANITY-CHECK-DEPLOYMENT: OK" > ../archives/deployment_status.log') def subprocess_run_w_echo(cmd): logging.debug(cmd)