diff --git a/ci-scripts/Jenkinsfile-Bracket-Test b/ci-scripts/Jenkinsfile-Bracket-Test new file mode 100644 index 0000000000000000000000000000000000000000..bee78998a8d75cff2d549dfe2ed5b065c29c5020 --- /dev/null +++ b/ci-scripts/Jenkinsfile-Bracket-Test @@ -0,0 +1,270 @@ +#!/bin/groovy +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +//------------------------------------------------------------------------------- + +// Location of the CN executor node +def cn_ci_host = params.Host_CN_CI_Server + +// for lock +def cn_ci_resource = params.DockerContainers +def ds_tester_ci_resource = params.DsTester + +// Location of the 2nd CN executor +def new_host_flag = false +def new_host = "" +def new_host_user = "" + +// Location of the CN tester +def dsT_host_flag = false +def dsT_host = "" +def dsT_host_user = "" +def dsT_host_ip_addr = "" + +// Flags +def scmEvent = false +def upstreamEvent = false +def deployed = true + +// Default tag --> could be passed on by upstream job or by PR content +def nrfTag = params.nrfTag + +//------------------------------------------------------------------------------- +// Pipeline start +pipeline { + agent { + label cn_ci_host + } + options { + disableConcurrentBuilds() + timestamps() + ansiColor('xterm') + lock(cn_ci_resource) + } + stages { + stage ('Verify Parameters') { + steps { + script { + echo '\u2705 \u001B[32mVerify Parameters\u001B[0m' + + JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"' + JOB_TIMESTAMP = JOB_TIMESTAMP.trim() + + if (params.Host_CN_CI_2nd_Server_Flag != null) { + new_host_flag = params.Host_CN_CI_2nd_Server_Flag + if (new_host_flag) { + new_host = params.Host_CN_CI_2nd_Server + new_host_user = params.Host_CN_CI_2nd_Server_Login + echo "1st Node is ${NODE_NAME}" + echo "2nd Node is ${new_host}" + } else { + echo "Node is ${NODE_NAME}" + } + } else { + echo "Node is ${NODE_NAME}" + } + if (params.DS_Tester_Server_Flag != null) { + dsT_host_flag = params.DS_Tester_Server_Flag + if (dsT_host_flag) { + def allParametersPresent = true + if (params.DS_Tester_Server_Name == null) { + allParametersPresent = false + } else { + dsT_host = params.DS_Tester_Server_Name + } + if (params.DS_Tester_Server_Login == null) { + allParametersPresent = false + } else { + dsT_host_user = params.DS_Tester_Server_Login + } + if (params.DS_Tester_Server_IP_Addr == null) { + allParametersPresent = false + } else { + dsT_host_ip_addr = params.DS_Tester_Server_IP_Addr + } + if (allParametersPresent) { + echo "DS Tester is on ${dsT_host}" + } else { + echo "Some DS Tester parameters are missing!" + sh "./ci-scripts/fail.sh" + } + } + } + + // Find out the cause of the trigger + for (cause in currentBuild.getBuildCauses()) { + if (cause.toString() ==~ /.*UpstreamCause.*/) { + upstreamEvent = true + //} else { + // scmEvent = true + } + } + if (upstreamEvent) { + if (params.NRF_TAG != null) { + nrfTag = params.NRF_TAG + echo "Upstream Job passed NRF_TAG to use: ${nrfTag}" + } + sh "git clean -x -d -f > /dev/null 2>&1" + sh "git fetch --prune > /dev/null 2>&1" + } + sh "mkdir -p archives DS-TEST-RESULTS" + // Verify that the images are available + try { + sh 'echo "OAI_NRF_TAG: oai-nrf:' + nrfTag +'" > archives/oai_nrf_image_info.log' + sh 'docker image inspect --format=\'Size = {{.Size}} bytes\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log' + sh 'docker image inspect --format=\'Date = {{.Created}}\' oai-nrf:' + nrfTag + ' >> archives/oai_nrf_image_info.log' + } catch (Exception e) { + error "OAI NRF Image tag to test does not exist!" + } + } + } + } + stage ('Deploy NRF-U18') { + steps { + script { + echo '\u2705 \u001B[32mDeploy NRF using Docker-Compose\u001B[0m' + // Prepare all needed files for docker-compose + // First put all correct tags to test + sh 'sed -e "s#NRF_IMAGE_TAG#' + nrfTag + '#" ci-scripts/dsTesterDockerCompose/docker-compose.tplt > ci-scripts/dsTesterDockerCompose/docker-compose.yml' + dir('ci-scripts/dsTesterDockerCompose') { + sh 'docker-compose up -d > ../../archives/compose_nrf_up.log 2>&1' + sh 'sleep 60' + // Do a check on number of healthy containers + // 1 == nrf + ret = sh returnStdout: true, script: 'docker-compose ps -a | grep -v unhealthy | grep -c healthy || true' + ret = ret.trim() + if (ret != '1') { + error "Deployment went wrong!" + } + } + } + } + post { + always { + script { + // Do docker logs to recover the configuration results + try { + sh 'docker inspect --format=\'STATUS: {{.State.Health.Status}}\' cicd-oai-nrf > archives/nrf_config.log' + } catch (Exception e) { + sh 'echo "STATUS: KO" >> archives/nrf_config.log' + } + } + } + success { + script { + sh 'echo "DEPLOYMENT: OK" > archives/deployment_status.log' + sh 'python3 ./ci-scripts/routeCheck.py --mode=Add --userName=' + dsT_host_user + ' --hostName=' + dsT_host + } + } + unsuccessful { + script { + sh 'echo "DEPLOYMENT: KO" > archives/deployment_status.log' + deployed = false + } + } + } + } + stage ('Bracket Testing with dsTester') { + when { expression {dsT_host_flag} } + steps { + lock (ds_tester_ci_resource) { + script { + echo '\u2705 \u001B[32mTesting with DS Tester\u001B[0m' + if (fileExists("dstester")) { + sh "rm -Rf dstester > /dev/null 2>&1" + } + sh "mkdir -p dstester" + dir ('dstester') { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.dsTesterGitLabRepository_Credentials}", usernameVariable: 'git_username', passwordVariable: 'git_token'] + ]) { + sh "git clone https://${git_username}:${git_token}@github.com/OPENAIRINTERFACE/chasseur.git . > ../git_clone.log 2>&1" + sh "git checkout edge >> ../git_clone.log 2>&1" + dir ('jenkins') { + try { + sh "python3 ./dogmatix-agent.py -f ./suits/dc/nrf-bracket.yaml -d true | tee ../../DS-TEST-RESULTS/dsTester_Summary.txt" + } catch (Exception e) { + currentBuild.result = 'FAILURE' + echo "dsTester Running FAILED" + } + } + } + } + sh "python3 ./ci-scripts/toCheckDSTesterResult.py" + } + } + } + post { + always { + script { + // Copy the pcap and log from the container + sh "mkdir -p archives/pcaps archives/logs" + try { + sh 'docker cp cicd-oai-nrf:/tmp/nrf.pcap archives/pcaps/oai_nrf.pcap' + sh 'docker cp cicd-oai-nrf:/tmp/nrf.log archives/logs/oai_nrf.log' + } catch (Exception e) { + sh 'echo "Error in copying pcap & log from the container"' + } + } + } + } + } + stage ('Undeploy NRF') { + steps { + script { + echo '\u2705 \u001B[32mUn-Deploy NRF\u001B[0m' + sh 'python3 ./ci-scripts/routeCheck.py --mode=Delete --userName=' + dsT_host_user + ' --hostName=' + dsT_host + dir('ci-scripts/dsTesterDockerCompose') { + sh 'docker-compose down > ../../archives/compose_normal_down.log 2>&1' + } + } + } + } + } + post { + always { + script { + // Get logs if deployment fails + if (deployed != true) { + sh 'mkdir -p archives/logs' + sh 'docker logs cicd-oai-nrf > archives/logs/oai_nrf.log' + } + // Remove any leftover containers/networks + sh 'python3 ./ci-scripts/routeCheck.py --mode=Delete --userName=' + dsT_host_user + ' --hostName=' + dsT_host + dir('ci-scripts/dsTesterDockerCompose') { + sh 'docker-compose down > ../../archives/compose_l_down.log 2>&1' + } + // Generating the HTML report + sh 'python3 ./ci-scripts/bracketTestHTMLReport.py --job_name=' + JOB_NAME + ' --job_id=' + BUILD_ID + ' --job_url=' + BUILD_URL + + // Zipping all archived log files + sh "zip -r -qq cn5g_nrf_docker_bt_logs.zip archives DS-TEST-RESULTS" + if (fileExists('cn5g_nrf_docker_bt_logs.zip')) { + archiveArtifacts artifacts: 'cn5g_nrf_docker_bt_logs.zip' + } + if (fileExists('test_results_oai_nrf_bt.html')) { + archiveArtifacts artifacts: 'test_results_oai_nrf_bt.html' + } + } + } + } +} diff --git a/ci-scripts/Jenkinsfile-GitLab-Docker b/ci-scripts/Jenkinsfile-GitLab-Docker index b68a7dd114aee103f9be37e9a63520b8c42d4254..c8dbca57c535ea147ab83f7df0b7f28c05851253 100644 --- a/ci-scripts/Jenkinsfile-GitLab-Docker +++ b/ci-scripts/Jenkinsfile-GitLab-Docker @@ -338,6 +338,35 @@ pipeline { } } } + stage ('Bracket Testing with DsTester') { + steps { + script { + gitlabCommitStatus(name: "Bracket Test with DsTester") { + localStatus = build job: params.NRF_BT_PipelineName, + parameters: [ + string(name: 'NRF_TAG', value: String.valueOf(nrf_tag)) + ], propagate: false + localResult = localStatus.getResult() + + if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Bracket Test Job is OK" + } else { + echo "Bracket Test Job is KO" + sh "ci-scripts/fail.sh" + } + } + } + } + post { + always { + script { + copyArtifacts(projectName: params.AMF_BT_PipelineName, + filter: '*_results_oai_nrf_bt.html', + selector: lastCompleted()) + } + } + } + } stage ('Testing in CN-5G-FED environment') { steps { script { @@ -439,6 +468,10 @@ pipeline { sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' *_results_oai_cn5g.html" sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_TEMPLATE#${env.gitlabMergeRequestTitle}#' *_results_oai_cn5g.html" } + if (fileExists('test_results_oai_nrf_bt.html')) { + sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_LINK#${gitlabMergeRequestLink}#g' *_results_oai_nrf_bt.html" + sh "sed -i -e 's#TEMPLATE_MERGE_REQUEST_TEMPLATE#${env.gitlabMergeRequestTitle}#' *_results_oai_nrf_bt.html" + } } else { sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${GIT_BRANCH} --git_src_commit=${GIT_COMMIT}" } @@ -451,6 +484,10 @@ pipeline { sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' *_results_oai_cn5g.html" archiveArtifacts artifacts: '*_results_oai_cn5g.html' } + if (fileExists('test_results_oai_nrf_bt.html')) { + sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' *_results_oai_nrf_bt.html" + archiveArtifacts artifacts: '*_results_oai_nrf_bt.html' + } // Sending email to commiter if (params.sendToCommitterEmail != null) { diff --git a/ci-scripts/dsTesterDockerCompose/docker-compose.tplt b/ci-scripts/dsTesterDockerCompose/docker-compose.tplt new file mode 100644 index 0000000000000000000000000000000000000000..cd261e6b6e27f651d83c652322f183ff0d4e31f7 --- /dev/null +++ b/ci-scripts/dsTesterDockerCompose/docker-compose.tplt @@ -0,0 +1,40 @@ +version: '3.8' +services: + cicd_oai_nrf: + container_name: cicd-oai-nrf + image: oai-nrf:NRF_IMAGE_TAG + ports: + - 80 + - 9090 + command: > + bash -c "nohup tshark -i eth0 -w /tmp/nrf.pcap 2>&1 > /dev/null & + /openair-nrf/bin/oai_nrf -c /openair-nrf/etc/nrf.conf -o | tee /tmp/nrf.log 2>&1 + " + cap_add: + - NET_ADMIN + environment: + - TZ=Europe/Paris + - NRF_INTERFACE_NAME_FOR_SBI=eth0 + - NRF_INTERFACE_PORT_FOR_SBI=80 + - NRF_INTERFACE_HTTP2_PORT_FOR_SBI=9090 + - NRF_API_VERSION=v1 + - INSTANCE=0 + - PID_DIRECTORY=/var/run + networks: + cicd_public_net: + ipv4_address: 192.168.61.195 + volumes: + - ./nrf-healthy-check.sh:/openair-nrf/bin/nrf-healthy-check.sh + healthcheck: + test: /bin/bash -c "/openair-nrf/bin/nrf-healthy-check.sh" + interval: 10s + timeout: 5s + retries: 5 + +networks: + cicd_public_net: + name: cicd-oai-public-net + driver: bridge + ipam: + config: + - subnet: 192.168.61.192/26 diff --git a/ci-scripts/dsTesterDockerCompose/nrf-healthy-check.sh b/ci-scripts/dsTesterDockerCompose/nrf-healthy-check.sh new file mode 100755 index 0000000000000000000000000000000000000000..d4b59a3c266e8d07f599109a01a18ef882d30d36 --- /dev/null +++ b/ci-scripts/dsTesterDockerCompose/nrf-healthy-check.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +STATUS=0 + +RESULT=$(ps aux | grep -v nohup || true) +SUB='/openair-nrf/bin/oai_nrf -c /openair-nrf/etc/nrf.conf -o' +if [[ $RESULT =~ $SUB ]]; then + STATUS=0 +else + STATUS=-1 +fi + +exit $STATUS diff --git a/ci-scripts/routeCheck.py b/ci-scripts/routeCheck.py new file mode 100644 index 0000000000000000000000000000000000000000..d45601354f7dd831a3656004f0f6d0cb707a8901 --- /dev/null +++ b/ci-scripts/routeCheck.py @@ -0,0 +1,148 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- + +import os +import re +import sys +import subprocess +import yaml + +class IpRouteCheck(): + def __init__(self): + self.mode = '' + self.userName = '' + self.hostName = '' + self.subnet = '' + self.gatwayIp = '' + self.interfaceName = '' + + def getSubnet(self): + cmd = "egrep 'subnet' ci-scripts/dsTesterDockerCompose/docker-compose.yml" + ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') + if ret.returncode == 0: + if ret.stdout is not None: + result = re.search("(?:[0-9]{1,3}[.]){3}[0-9]{1,3}/[0-9]{1,2}", ret.stdout.strip()) + if result is not None: + self.subnet = result.group(0) + #print("found subnet:", self.subnet) + else: + print("subnet not found in docker compose") + sys.exit(-1) + else: + print("docker-compose file not found") + sys.exit(-1) + + def getGatwayIp(self): + cmd = "ifconfig | grep 192.168.18" + ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') + if ret.returncode == 0: + result = re.search("inet (?:[0-9]{1,3}[.]){3}[0-9]{1,3}", ret.stdout.strip()) + self.gatwayIp = result.group(0) + #print("Gatway IP Address:", self.gatwayIp) + else: + print("No Gatway IP address starts with 196.168.18") + sys.exit(-1) + + def routeCheck(self): + # Get the interface name + cmd = "ip route | grep 192.168.18.0/24 | awk {'print $3'}" + ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8') + #print('interface name:', ret.stdout.strip()) + self.interfaceName = ret.stdout.strip() + # Check whether or not ip route exist + cmd = f'ip route | grep -c "{self.subnet}"' + ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8') + if ret.stdout is not None: + if ret.stdout.strip() == '1': + #print('Route exits') + if IPRC.mode == 'Delete': + IPRC.routeDel() + else: + sys.exit(0) + else: + #print("Route not found") + if IPRC.mode == 'Add': + IPRC.routeAdd() + else: + sys.exit(0) + + def routeAdd(self): + # Add the route + cmd = f"sudo ip route add {self.subnet} via {self.gatwayIp} dev {self.interfaceName}" + ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8') + print("Added ip route") + + def routeDel(self): + # Delete the route + cmd = f"sudo ip route del {self.subnet} via {self.gatwayIp} dev {self.interfaceName}" + ret = subprocess.run(f'ssh {self.userName}@{self.hostName} {cmd} || true', shell=True, stdout=subprocess.PIPE, encoding='utf-8') + print("Deleted ip route") + +def Usage(): + print('----------------------------------------------------------------------------------------------------------------------') + print('routecheck.py') + print(' Add and Delete the ip route on the Server.') + print('----------------------------------------------------------------------------------------------------------------------') + print('Usage: python3 routecheck.py [options]') + print(' --help Show this help.') + print('---------------------------------------------------------------------------------------------- Mandatory Options -----') + print(' --mode=[Add/Delete]') + print(' --userName=[server userName where to add/delete]') + print(' --hostName=[server hostName where to add/delete]') + print('------------------------------------------------------------------------------------------------- Actions Syntax -----') + print('python3 routeCheck.py --mode=Add [Mandatory Options]') + print('python3 routeCheck.py --mode=Delete [Mandatory Options]') + + +#-------------------------------------------------------------------------------------------------------- +# +# Start of main +# +#-------------------------------------------------------------------------------------------------------- + +argvs = sys.argv + +IPRC = IpRouteCheck() + +while len(argvs) > 1: + myArgv = argvs.pop(1) + if re.match('^\-\-help$', myArgv, re.IGNORECASE): + Usage() + sys.exit(0) + elif re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-mode=(.+)$', myArgv, re.IGNORECASE) + IPRC.mode = matchReg.group(1) + elif re.match('^\-\-userName=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-userName=(.+)$', myArgv, re.IGNORECASE) + IPRC.userName = matchReg.group(1) + elif re.match('^\-\-hostName=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-hostName=(.+)$', myArgv, re.IGNORECASE) + IPRC.hostName = matchReg.group(1) + else: + sys.exit('Invalid Parameter: ' + myArgv) + +if IPRC.mode == '' or IPRC.userName == '' or IPRC.hostName == '': + sys.exit('Missing Parameter in job description') + +IPRC.getSubnet() +IPRC.getGatwayIp() +IPRC.routeCheck() diff --git a/ci-scripts/toCheckDSTesterResult.py b/ci-scripts/toCheckDSTesterResult.py new file mode 100644 index 0000000000000000000000000000000000000000..46e97b10b9c610c07dd8ea29c5af1a210a36a6b3 --- /dev/null +++ b/ci-scripts/toCheckDSTesterResult.py @@ -0,0 +1,53 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- + +import re +import sys +import subprocess +import yaml +import os +import time + +locexist = False +cwd = os.getcwd() +try: + with open(cwd + '/DS-TEST-RESULTS/dsTester_Summary.txt') as f: + for line in f: + if re.search('Result file is available here', str(line)): + result = re.search('(?:\/.+?\/)(.+?)(?:\/.+)', str(line)) + if result: + result1 = re.search('^(.*/)([^/]*)$', str(result.group(0))) + subprocess.check_output(f'cp {result1.group(1)}* DS-TEST-RESULTS/', stderr=subprocess.STDOUT, shell=True, universal_newlines=True) + result2 = re.search('.*\/(.+)$', str(result.group(0))) + filename = result2.group(1) + locexist = True +except IOError: + sys.exit("File not accessible to check DSTester Summary: DS-TEST-RESULTS/dsTester_Summary.txt") + +if locexist: + try: + with open(cwd + f'/DS-TEST-RESULTS/{filename}') as f: + data = yaml.load(f) + if data["final-result"] == 'fail': + sys.exit('DsTester final result FAILED') + except IOError: + sys.exit(f'File not accessible to check DSTester result: DS-TEST-RESULTS/{filename}')