#! /usr/bin/python #/* # * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more # * contributor license agreements. See the NOTICE file distributed with # * this work for additional information regarding copyright ownership. # * The OpenAirInterface Software Alliance licenses this file to You under # * the OAI Public License, Version 1.0 (the "License"); you may not use this file # * except in compliance with the License. # * You may obtain a copy of the License at # * # * http://www.openairinterface.org/?page_id=698 # * # * Unless required by applicable law or agreed to in writing, software # * distributed under the License is distributed on an "AS IS" BASIS, # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # * See the License for the specific language governing permissions and # * limitations under the License. # *------------------------------------------------------------------------------- # * For more information about the OpenAirInterface (OAI) Software Alliance: # * contact@openairinterface.org # */ # \author Rohit Gupta - Benoit ROBERT (benoit.robert@syrtem.com) # \version 0.1 # @ingroup _test # \Changelog # 2016-11-18 : # - Add progess bar during test execution update_progress() import tempfile import threading import sys import traceback import wave import os import time import datetime import getpass import math #from time import clock import xml.etree.ElementTree as ET import re #from dict2xml import dict2xml as xmlify from colorama import Fore, Back, Style import numpy as np import log from openair import * import paramiko import subprocess import commands sys.path.append('/opt/ssh') sys.path.append(os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/tools/')) from lib_autotest import * import ssh from ssh import SSHSession import argparse # update_progress() : Displays or updates a console progress bar ## Accepts a float between 0 and 1. Any int will be converted to a float. ## A value under 0 represents a 'halt'. ## A value at 1 or bigger represents 100% def update_progress(progress, prefix_string): barLength = 20 # Modify this to change the length of the progress bar status = "" #print "progress = "+ str(progress) if isinstance(progress, int): progress = float(progress) if not isinstance(progress, float): progress = 0 status = "error: progress var must be float\r\n" if progress < 0: progress = 0 status = "Halt...\r\n" if progress >= 1: progress = 1 status = "Done...\r\n" block = int(round(barLength*progress)) text = "\r"+prefix_string+" [{0}] {1}% {2}".format( "="*block + " "*(barLength-block), progress*100, status) sys.stdout.write(text) sys.stdout.flush() def exit_prog(exit_val): print(Style.RESET_ALL), sys.exit(exit_val) # \brief write a string to a file # \param filename name of file # \param string string to write # \mode file opening mode (default=write) def write_file(filename, string, mode="w"): text_file = open(filename, mode) text_file.write(string) text_file.close() # \brief function to check if test case passed throughput test # \param filename name of file which has throughput results (usually from iperf -s ... # \param min_tput minimum throughput # \param max_tuput maximum throughput # \param average average throughput # \param min_duration minimum duration of throughput #The throughput values found in file must be higher than values from from arguments 2,3,4,5 #The function returns True if throughput conditions are saisfied else it returns fails def tput_test(filename, min_tput, max_tput, average, min_duration): if os.path.exists(filename): with open (filename, "r") as myfile: data=myfile.read() p=re.compile('(\d*.\d*) Mbits/sec') array=p.findall(data) array = [ float(x) for x in array ] duration = array.__len__() if duration !=0: min_list = min(array) max_list = max(array) average_list = np.mean(array) else: min_list = 0 max_list = 0 average_list=0 tput_string=' ( '+ "min=%0.2f" % min_list + ' Mbps / ' + "max=%0.2f" % max_list + ' Mbps / ' + "avg=%0.2f" % average_list + ' Mbps / ' + "dur=%0.2f" % duration + ' s) ' if (min_list >= min_tput and max_list >= max_tput and average_list >= average and duration >= min_duration): return True , tput_string else: return False , tput_string else: return False , tput_string # \brief Convert string to float or return None if there is exception def try_convert_to_float(string, fail=None): try: return float(string) except Exception: return fail; # \brief get throughput statistics from log file # \param search_expr search expression found in test_case_list.xml file # \param logfile_traffic logfile which has traffic statistics def tput_test_search_expr (search_expr, logfile_traffic): result=0 tput_string='' if search_expr !='': if search_expr.find('throughput_test')!= -1 : p= re.compile('min\s*=\s*(\d*.\d*)\s*Mbits/sec') min_tput=p.findall(search_expr) if min_tput.__len__()==1: min_tput = min_tput[0] else: min_tput = None p= re.compile('max\s*=\s*(\d*.\d*)\s*Mbits/sec') max_tput=p.findall(search_expr) if max_tput.__len__()==1: max_tput = max_tput[0] else: max_tput = None p= re.compile('average\s*=\s*(\d*.\d*)\s*Mbits/sec') avg_tput=p.findall(search_expr) if avg_tput.__len__()==1: avg_tput=avg_tput[0] else: avg_tput = None p= re.compile('duration\s*=\s*(\d*.\d*)\s*s') duration=p.findall(search_expr) if duration.__len__()==1: duration = duration[0] else: duration = None min_tput = try_convert_to_float(min_tput) max_tput = try_convert_to_float(max_tput) avg_tput = try_convert_to_float(avg_tput) duration = try_convert_to_float(duration) if (min_tput != None and max_tput != None and avg_tput != None and duration != None ): result, tput_string = tput_test(logfile_traffic, min_tput, max_tput, avg_tput, duration) else: result=1 return result, tput_string # \brief function to copy files to/from remote machine # \param username user with which to make sftp connection # \param password password of user # \param hostname host to connect # \ports port of remote machine on which server is listening # \paramList This is list of operations as a set {operation: "get/put", localfile: "filename", remotefile: "filename" # \param logfile Ignored currently and set once at the beginning of program def sftp_module (username, password, hostname, ports, paramList,logfile): #localD = localfile #remoteD = remotefile #fd, paramiko_logfile = tempfile.mkstemp() #res = os.close(fd ) #paramiko logfile path should not be changed with multiple calls. The logs seem to in first file regardless error = "" #The lines below are outside exception loop to be sure to terminate the test case if the network connectivity goes down or there is authentication failure transport = paramiko.Transport((hostname, ports)) transport.connect(username = username, password = password) sftp = paramiko.SFTPClient.from_transport(transport) # index =0 for param in paramList: try: operation = param["operation"] localD = param["localfile"] remoteD = param["remotefile"] if operation == "put": sftp.put(remotepath=remoteD, localpath=localD) elif operation == "get": sftp.get(remotepath=remoteD, localpath=localD) else : print "sftp_module: unidentified operation:<" + operation + "> Exiting now" print "hostname = " + hostname print "ports = " + ports print "localfile = " + localD print "remotefile = " + remoteD print "operation = " + operation sys.exit() except Exception, e: error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e) error = error + '\n username = ' + username + '\n hostname = ' + hostname + '\n localfile = ' + localD + '\n remotefile = ' + remoteD + '\n operation = ' + operation + '\nlogfile = ' + logfile + '\n ports = ' + str(ports) + '\n' error = error + traceback.format_exc() print error sftp.close() transport.close() res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> starts...\' >> ' + logfile + ' 2>&1 ') res = os.system('cat ' + paramiko_logfile + ' >> ' + logfile + ' 2>&1 \n') write_file(logfile, error, "a") res = os.system('\n echo \'SFTP Module Log for Machine: <' + hostname + '> ends...\' >> ' + logfile + ' 2>&1 \n') # \brief bash script stub put at the end of scripts to terminate it # \param timeout_cmd terminate script after timeout_cmd seconds # \param terminate_missing_procs if True terminate all the processes launched by script if one of them terminates prematurely (due to error) def finalize_deploy_script (timeout_cmd, terminate_missing_procs='False'): cmd = 'declare -i timeout_cmd='+str(timeout_cmd) + '\n' if terminate_missing_procs == 'True': cmd = cmd + """ #The code below checks if one the processes launched in background has crashed. #If it does, then the code below terminates all the child processes created by this script declare -i wakeup_interval=1 declare -i step=0 echo \"Array pid = ${array_exec_pid[@]}\" while [ "$step" -lt "$timeout_cmd" ] do declare -i break_while_loop=0 #Iterate over each process ID in array_exec_pid for i in "${array_exec_pid[@]}" do numchild=`pstree -p $i | perl -ne 's/\((\d+)\)/print " $1"/ge' |wc -w` echo "PID = $i, numchild = $numchild" if [ "$numchild" -eq "0" ] ; then echo "Process ID $i has finished unexpectedly. Now preparing to kill all the processes " break_while_loop=1 break fi done if [ "$break_while_loop" -eq "1" ] ; then break fi step=$(( step + wakeup_interval )) sleep $wakeup_interval done echo "Final time step (Duration of test case) = $step " date """ else: #We do not terminate the script if one of the processes has existed prematurely cmd = cmd + 'sleep ' + str(timeout_cmd) + ' ; date \n' return cmd # \brief run python script and update config file params of test case # \param oai module with already open connection # \param config_string config string taken from xml file # \param logdirRepo directory of remote repository # \param python_script python script location def update_config_file(oai, config_string, logdirRepo, python_script): cmd="" if config_string : stringArray = config_string.splitlines() #python_script = '$OPENAIR_DIR/targets/autotests/tools/search_repl.py' for string in stringArray: #split the string based on space now string1=string.split() cmd = cmd + 'python ' + python_script + ' ' + logdirRepo+'/'+string1[0] + ' ' + string1[1] + ' '+ string1[2] + '\n' #cmd = cmd + 'perl -p -i -e \'s/'+ string1[1] + '\\s*=\\s*"\\S*"\\s*/' + string1[1] + ' = "' + string1[2] +'"' + '/g\' ' + logdirRepo + '/' +string1[0] + '\n' return cmd #result = oai.send_recv(cmd) # \brief thread safe sshsession wrapper due to occasional connection issues with ssh # \param machine name of machine # \param username user login for remote machine # \param key_file file name which has keys to enable passwordless login # \param password password for remote machine # \param logdir_remote remote directory # \param logdir_local_base local directory # \param operation operation to perform (get_all, put_all) transfers recursively for directories def SSHSessionWrapper(machine, username, key_file, password, logdir_remote, logdir_local_base, operation): max_tries = 10 i=0 while i <= max_tries: i = i +1 try: ssh = SSHSession(machine , username, key_file, password) if operation == "get_all": ssh.get_all(logdir_remote , logdir_local_base) elif operation == "put_all": ssh.put_all(logdir_local_base, logdir_remote ) else: print "Error: Uknown operation in SSHSessionWrapper. Exiting now..." sys.exit(1) break except Exception, e: error='' error = error + ' In Class = function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e) error = error + '\n username = ' + username + '\n machine = ' + machine + '\n logdir_remote = ' + logdir_remote + '\n logdir_local_base = ' + logdir_local_base error = error + traceback.format_exc() print error print " Retrying again in 1 seconds" time.sleep(1) print "Continuing ..." if i ==max_tries: print "Fatal Error: Max no of retries reached. Exiting now..." sys.exit(1) # \briefFunction to clean old programs that might be running from earlier execution # \param oai - parameter for making connection to machine # \parm programList list of programs that must be terminated before execution of any test case # \param CleanUpAluLteBox program to terminate AlU Bell Labs LTE Box # \param ExmimoRfStop String to stop EXMIMO card (specified in test_case_list.xml) def cleanOldPrograms(oai, programList, CleanUpAluLteBox, ExmimoRfStop, logdir, logdirOAI5GRepo): cmd = 'killall -9 ' + programList result = oai.send(cmd, True) # print "\t\t > "+cmd # print "\t\t < "+result # print "Killing old programs on ..." + result programArray = programList.split() programListJoin = '|'.join(programArray) cmd = " ( date ;echo \"Starting cleaning old programs.. \" ; dmesg|tail ; echo \"Current disk space.. \" ; df -h )>& " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync" result=oai.send_recv(cmd) # print "\t\t > "+cmd # print "\t\t < "+result cmd = cleanupOldProgramsScript + ' ' + '\''+programListJoin+'\'' #result = oai.send_recv(cmd) #print result result = oai.send_expect_false(cmd, 'Match found', False) # print "\t\t > "+cmd # print "\t\t < "+result # print "Looking for old programs..." + result res=oai.send_recv(CleanUpAluLteBox, True) # print "\t\t > "+CleanUpAluLteBox # print "\t\t < "+res cmd= " echo \"Starting EXmimoRF Stop... \" >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync "; result = oai.send_recv(cmd) # print "\t\t > "+cmd # print "\t\t < "+result cmd = "( " + "cd " + logdirOAI5GRepo + " ; source oaienv ; " + ExmimoRfStop + " ) >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync " # print "cleanoldprograms cmd = " + cmd res=oai.send_recv(cmd, False, timeout=600) # print "\t\t > "+cmd # print "\t\t < "+res cmd= " echo \"Stopping EXmimoRF Stop... \" >> " + logdir + "/oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync "; result = oai.send_recv(cmd) # print "\t\t > "+cmd # print "\t\t < "+result #res = oai.send_recv(ExmimoRfStop, False) cmd = " ( date ;echo \"Finished cleaning old programs.. \" ; dmesg | tail)>> $HOME/.oai_test_setup_cleanup.log.`hostname` 2>&1 ; sync" res=oai.send_recv(cmd) # print "\t\t > "+cmd # print "\t\t < "+res # \brief Class thread to launch a generic command on remote machine # \param threadID number of thread (for book keeping) # \param threadname string of threadname (for book keeping) # \param machine machine name on which to run the command # \param username username with which to login # \param password password with which to login # \param cmd command as a string to run on remote machine # \parma sudo if True sudo is set # \param timeout timeout of command in seconds class oaiThread (threading.Thread): def __init__(self, threadID, threadname, machine, username, password, cmd, sudo, timeout): threading.Thread.__init__(self) self.threadID = threadID self.threadname = threadname self.machine = machine self.username = username self.password = password self.cmd = cmd self.sudo = sudo self.timeout = timeout def run(self): try: oai = openair('localdomain',self.machine) oai.connect(self.username, self.password) # print "Starting " + self.threadname + " on machine " + self.machine result = oai.send_recv(self.cmd, self.sudo, self.timeout) #print "result = " + result #print "Exiting " + self.threadname oai.disconnect() except Exception, e: error='' error = error + ' In class oaiThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e) error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n timeout = ' + str(self.timeout) + '\n machine = ' + self.machine + '\n cmd = ' + self.cmd + '\n timeout = ' + str(self.timeout) + '\n username = ' + self.username + '\n' error = error + traceback.format_exc() print error # \brief This class runs test cases with class {execution, compilatation} # \param threadID number of thread (for book keeping) # \param name string of threadname (for book keeping) # \param machine machine name on which to run the command # \param logdirOAI5GRepo directory on remote machine which as openairinterface5g repo installed # \param testcasename name of test case to run on remote machine # \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml) # \param user username with which to login # \param password password with which to login # \param timeout timeout of command in seconds # \param ExmimoRfStop command to stop EXMIMO Card class testCaseThread_generic (threading.Thread): def __init__(self, threadID, name, machine, logdirOAI5GRepo, testcasename,oldprogramList, CleanupAluLteBox, user, password, timeout, ExmimoRfStop): threading.Thread.__init__(self) self.threadID = threadID self.name = name self.testcasename = testcasename self.timeout = timeout self.machine = machine self.logdirOAI5GRepo = logdirOAI5GRepo self.oldprogramList = oldprogramList self.CleanupAluLteBox = CleanupAluLteBox self.password=password self.ExmimoRfStop = ExmimoRfStop self.user = user def run(self): try: mypassword='' #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E ' addpass = 'echo \'' + mypassword + '\' | ' #user = getpass.getuser() print "Starting test case : " + self.testcasename + " On machine " + self.machine + " timeout = " + str(self.timeout) oai = openair('localdomain',self.machine) oai.connect(self.user, self.password) #cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop) logdir_local = os.environ.get('OPENAIR_DIR') logdir_local_testcase = logdir_local +'/cmake_targets/autotests/log/'+ self.testcasename logdir_local_base = logdir_local +'/cmake_targets/autotests/log/' logdir_remote_testcase = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/' + self.testcasename logdir_remote = self.logdirOAI5GRepo + '/cmake_targets/autotests/log/' logfile_task_testcasename = logdir_local_testcase + '/test_task' + '_' + self.testcasename + '_.log' logfile_task_testcasename_out = logdir_remote + '/test_task_out' + '_' + self.testcasename + '_.log' #print "logdir_local_testcase = " + logdir_local_testcase #print "logdir_remote_testcase = " + logdir_remote_testcase #if os.path.exists(logdir_local_testcase) == True : # os.removedirs(logdir_local_testcase) #os.mkdir(logdir_local_testcase) os.system("rm -fr " + logdir_local_testcase ) os.system("mkdir -p " + logdir_local_testcase) cmd = "mkdir -p " + logdir_remote_testcase res = oai.send_recv(cmd, False, self.timeout) #print "res = " + res cmd = "( cd " + self.logdirOAI5GRepo + " \n " cmd = cmd + "source oaienv \n" cmd = cmd + "$OPENAIR_DIR/cmake_targets/autotests/run_exec_autotests.bash --run-group \"" + self.testcasename + "\" -p \'\'" cmd = cmd + " ) >& " + logfile_task_testcasename_out + " ; " + "mkdir -p " + logdir_remote_testcase + "; mv " + logfile_task_testcasename_out + " " +logdir_remote_testcase #print "cmd = " + cmd res = oai.send_recv(cmd, False, self.timeout) #print "res = " + res #print "ThreadID = " + str(self.threadID) + "ThreadName: " + self.name + " testcasename: " + self.testcasename + "Execution Result = " + res write_file(logfile_task_testcasename, cmd, mode="w") #Now we copy all the remote files #ssh = SSHSession(self.machine , username=user, key_file=None, password=self.password) #ssh.get_all(logdir_remote_testcase , logdir_local_base) SSHSessionWrapper(self.machine, self.user, None, self.password, logdir_remote_testcase, logdir_local_base, "get_all") print "Finishing test case : " + self.testcasename + " On machine " + self.machine #cleanOldPrograms(oai, self.oldprogramList, self.CleanupAluLteBox, self.ExmimoRfStop) #oai.kill(user,mypassword) oai.disconnect() except Exception, e: error='' error = error + ' In Class = testCaseThread_generic, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e) error = error + '\n threadID = ' + str(self.threadID) + '\n threadName = ' + self.name + '\n testcasename = ' + self.testcasename + '\n machine = ' + self.machine + '\n logdirOAI5GRepo = ' + self.logdirOAI5GRepo + '\n' + '\n timeout = ' + str(self.timeout) + '\n user = ' + self.user error = error + traceback.format_exc() print error print "Continuing with next test case..." #sys.exit() # \bried function to run a command as a sudo # \param cmd command as a string # \param password password to be supplied def addsudo (cmd, password=""): cmd = 'echo \'' + password + '\' | sudo -S -E bash -c \' ' + cmd + '\' ' return cmd # \brief handler for executing test cases (compilation, execution) # \param name of testcase # \param threadListGeneric list of threads which are already running on remote machines # \param oldprogramList list of programs which must be terminated before running a test case # \param logdirOAI5GRepo directory on remote machine which as openairinterface5g repo installed # \param MachineList list of all machines on which generic test cases can be run # \param user username with which to login # \param password password with which to login # \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml) # \param timeout timeout of command in seconds # \param ExmimoRfStop command to stop EXMIMO Card def handle_testcaseclass_generic (testcasename, threadListGeneric, oldprogramList, logdirOAI5GRepo, MachineList, user, password, CleanupAluLteBox,timeout, ExmimoRfStop): try: mypassword=password MachineListFree=[] threadListNew=[] while MachineListFree.__len__() == 0 : MachineListBusy=[] MachineListFree=[] threadListNew=[] #first we need to find the list of free machines that we could run our test case if threadListGeneric.__len__() ==0 : #This means no thread is started yet MachineListFree = MachineList[:] else : for param in threadListGeneric : thread_id = param["thread_id"] machine = param["Machine"] testcasenameold = param["testcasename"] thread_id.join(1) if thread_id.isAlive() == True: threadListNew.append(param) print "thread_id is alive: testcasename: " + testcasenameold + " on machine "+ machine if machine not in MachineListBusy: MachineListBusy.append(machine) else : print "thread_id is finished: testcasename: " + testcasenameold + " on machine " + machine #threadListGeneric.remove(param) #if machine not in MachineListFree: # MachineListFree.append(machine) #Now we check if there is at least one free machine MachineListFree = MachineList[:] for machine in MachineListBusy: if machine in MachineListFree: MachineListFree.remove(machine) print "MachineListFree = " + ','.join(MachineListFree) print "MachineListBusy = " + ','.join(MachineListBusy) print "MachineList = " + ','.join(MachineList) machine = MachineListFree[0] thread = testCaseThread_generic(1,"Generic Thread_"+testcasename+"_"+ "machine_", machine, logdirOAI5GRepo, testcasename, oldprogramList, CleanupAluLteBox, user, password, timeout, ExmimoRfStop) param={"thread_id":thread, "Machine":machine, "testcasename":testcasename} thread.start() threadListNew.append(param) return threadListNew except Exception, e: error='' error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e) error = error + '\n testcasename = ' + testcasename + '\n logdirOAI5GRepo = ' + logdirOAI5GRepo + '\n MachineList = ' + ','.join(MachineList) + '\n timeout = ' + str(timeout) + '\n' + 'user = ' + user error = error + traceback.format_exc() print error print "Continuing..." #sys.exit(1) # \brief Blocking wait for all threads related to generic testcase execution, class (compilation and execution) # \param threadListGeneric list of threads which are running on remote machines # \param timeout time to wait on threads in seconds def wait_testcaseclass_generic_threads(threadListGeneric, timeout = 1): threadListGenericNew=[] for param in threadListGeneric: thread_id = param["thread_id"] machine = param["Machine"] testcasenameold = param["testcasename"] thread_id.join(timeout) if thread_id.isAlive() == True: threadListGenericNew.append(param) print "thread_id on machine: " + machine + " is still alive: testcasename: " + testcasenameold print " Exiting now..." sys.exit(1) else: print "thread_id on machine: " + machine + " is stopped: testcasename: " + testcasenameold #threadListGeneric.remove(param) return threadListGenericNew # \brief handler for executing test cases (lte-softmodem) # \param testcase name of testcase # \param oldprogramList list of programs which must be terminated before running a test case # \param logdirOAI5GRepo directory on remote machine which has openairinterface5g repo installed # \param logdirOpenaircnRepo directory on remote machine which has openair-cn repo installed # \param MachineList list of all machines on which test cases can be run # \param user username with which to login # \param password password with which to login # \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml) # \param ExmimoRfStop command to stop EXMIMO Card # \param nruns_lte-softmodem global parameter to override number of runs (nruns) within the test case def handle_testcaseclass_softmodem (testcase, oldprogramList, logdirOAI5GRepo , logdirOpenaircnRepo, MachineList, user, password, CleanUpAluLteBox, ExmimoRfStop, nruns_lte_softmodem, timeout_cmd): #We ignore the password sent to this function for secuirity reasons for password present in log files #It is recommended to add a line in /etc/sudoers that looks something like below. The line below will run sudo without password prompt # your_user_name ALL=(ALL:ALL) NOPASSWD: ALL mypassword='' #addsudo = 'echo \'' + mypassword + '\' | sudo -S -E ' addpass = 'echo \'' + mypassword + '\' | ' #user = getpass.getuser() testcasename = testcase.get('id') testcaseclass = testcase.findtext('class',default='') if timeout_cmd == '': timeout_cmd = testcase.findtext('TimeOut_cmd',default='') timeout_cmd = int(float(timeout_cmd)) #Timeout_thread is more than that of cmd to have room for compilation time, etc timeout_thread = timeout_cmd + 300 if nruns_lte_softmodem == '': nruns = testcase.findtext('nruns',default='') else: nruns = nruns_lte_softmodem nruns = int(float(nruns)) tags = testcase.findtext('tags',default='') RRHMachine = testcase.findtext('RRH',default='') RRH_compile_prog = testcase.findtext('RRH_compile_prog',default='') RRH_compile_prog_args = testcase.findtext('RRH_compile_prog_args',default='') RRH_pre_exec = testcase.findtext('RRH_pre_exec',default='') RRH_pre_exec_args = testcase.findtext('RRH_pre_exec_args',default='') RRH_main_exec = testcase.findtext('RRH_main_exec',default='') RRH_main_exec_args = testcase.findtext('RRH_main_exec_args',default='') RRH_terminate_missing_procs = testcase.findtext('RRH_terminate_missing_procs',default='False') eNBMachine = testcase.findtext('eNB',default='') eNB_config_file = testcase.findtext('eNB_config_file',default='') eNB_compile_prog = testcase.findtext('eNB_compile_prog',default='') eNB_compile_prog_args = testcase.findtext('eNB_compile_prog_args',default='') eNB_pre_exec = testcase.findtext('eNB_pre_exec',default='') eNB_pre_exec_args = testcase.findtext('eNB_pre_exec_args',default='') eNB_main_exec = testcase.findtext('eNB_main_exec',default='') eNB_main_exec_args = testcase.findtext('eNB_main_exec_args',default='') eNB_traffic_exec = testcase.findtext('eNB_traffic_exec',default='') eNB_traffic_exec_args = testcase.findtext('eNB_traffic_exec_args',default='') eNB_terminate_missing_procs = testcase.findtext('eNB_terminate_missing_procs',default='False') eNB_search_expr_true = testcase.findtext('eNB_search_expr_true','') if re.compile('\w+').match(eNB_search_expr_true) != None: eNB_search_expr_true = eNB_search_expr_true + ' duration=' + str(timeout_cmd-90) + 's' UEMachine = testcase.findtext('UE',default='') UE_config_file = testcase.findtext('UE_config_file',default='') UE_compile_prog = testcase.findtext('UE_compile_prog',default='') UE_compile_prog_args = testcase.findtext('UE_compile_prog_args',default='') UE_pre_exec = testcase.findtext('UE_pre_exec',default='') UE_pre_exec_args = testcase.findtext('UE_pre_exec_args',default='') UE_main_exec = testcase.findtext('UE_main_exec',default='') UE_main_exec_args = testcase.findtext('UE_main_exec_args',default='') UE_traffic_exec = testcase.findtext('UE_traffic_exec',default='') UE_traffic_exec_args = testcase.findtext('UE_traffic_exec_args',default='') UE_terminate_missing_procs = testcase.findtext('UE_terminate_missing_procs',default='False') UE_search_expr_true = testcase.findtext('UE_search_expr_true','') UE_stop_script = testcase.findtext('UE_stop_script','') if re.compile('\w+').match(UE_search_expr_true) != None: UE_search_expr_true = UE_search_expr_true + ' duration=' + str(timeout_cmd-90) + 's' EPCMachine = testcase.findtext('EPC',default='') EPC_config_file = testcase.findtext('EPC_config_file',default='') EPC_compile_prog = testcase.findtext('EPC_compile_prog',default='') EPC_compile_prog_args = testcase.findtext('EPC_compile_prog_args',default='') HSS_compile_prog = testcase.findtext('HSS_compile_prog',default='') HSS_compile_prog_args = testcase.findtext('HSS_compile_prog_args',default='') EPC_pre_exec= testcase.findtext('EPC_pre_exec',default='') EPC_pre_exec_args = testcase.findtext('EPC_pre_exec_args',default='') EPC_main_exec= testcase.findtext('EPC_main_exec',default='') EPC_main_exec_args = testcase.findtext('EPC_main_exec_args',default='') HSS_main_exec= testcase.findtext('HSS_main_exec',default='') HSS_main_exec_args = testcase.findtext('HSS_main_exec_args',default='') EPC_traffic_exec = testcase.findtext('EPC_traffic_exec',default='') EPC_traffic_exec_args = testcase.findtext('EPC_traffic_exec_args',default='') EPC_terminate_missing_procs = testcase.findtext('EPC_terminate_missing_procs',default='False') EPC_search_expr_true = testcase.findtext('EPC_search_expr_true','') if re.compile('\w+').match(EPC_search_expr_true) != None: EPC_search_expr_true = EPC_search_expr_true + ' duration=' + str(timeout_cmd-90) + 's' index_eNBMachine = MachineList.index(eNBMachine) index_UEMachine = MachineList.index(UEMachine) index_EPCMachine = MachineList.index(EPCMachine) cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR' oai_eNB = openair('localdomain', eNBMachine) oai_eNB.connect(user, password) res= oai_eNB.send_recv(cmd) oai_UE = openair('localdomain', UEMachine) oai_UE.connect(user, password) res = oai_eNB.send_recv(cmd) oai_EPC = openair('localdomain', EPCMachine) oai_EPC.connect(user, password) res = oai_eNB.send_recv(cmd) if RRHMachine != '': cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep OPENAIR' index_RRHMachine = MachineList.index(RRHMachine) oai_RRH = openair('localdomain', RRHMachine) oai_RRH.connect(user, password) res= oai_RRH.send_recv(cmd) #cleanOldPrograms(oai_eNB, oldprogramList, CleanUpAluLteBox, ExmimoRfStop) #cleanOldPrograms(oai_UE, oldprogramList, CleanUpAluLteBox, ExmimoRfStop) #cleanOldPrograms(oai_EPC, oldprogramList, CleanUpAluLteBox, ExmimoRfStop) logdir_local = os.environ.get('OPENAIR_DIR') if logdir_local is None: print "Environment variable OPENAIR_DIR not set correctly" sys.exit() #Make the log directory of test case #cmd = 'mkdir -p ' + logdir_eNB #result = oai_eNB.send_recv(cmd) #cmd = 'mkdir -p ' + logdir_UE #result = oai_UE.send_recv(cmd) #cmd = 'mkdir -p ' + logdir_EPC #result = oai_EPC.send_recv(cmd) print "Updating the config files for ENB/UE/EPC..." #updating the eNB/UE/EPC configuration file from the test case #update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo) #update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo) #update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo) test_result=1 test_result_string='' start_time=time.time() for run in range(0,nruns): run_result=1 run_result_string='' logdir_eNB = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run) logdir_RRH = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run) logdir_UE = logdirOAI5GRepo+'/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run) logdir_EPC = logdirOpenaircnRepo+'/TEST/autotests/log/'+ testcasename + '/run_' + str(run) logdir_local_testcase = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run) #Make the log directory of test case if RRHMachine != '': cmd = 'rm -fr ' + logdir_RRH + ' ; mkdir -p ' + logdir_RRH result = oai_RRH.send_recv(cmd) cmd = 'rm -fr ' + logdir_eNB + ' ; mkdir -p ' + logdir_eNB result = oai_eNB.send_recv(cmd) cmd = 'rm -fr ' + logdir_UE + ' ; mkdir -p ' + logdir_UE result = oai_UE.send_recv(cmd) cmd = 'rm -fr ' + logdir_EPC + '; mkdir -p ' + logdir_EPC result = oai_EPC.send_recv(cmd) cmd = ' rm -fr ' + logdir_local_testcase + ' ; mkdir -p ' + logdir_local_testcase result = os.system(cmd) if RRHMachine != '': logfile_compile_RRH = logdir_RRH + '/RRH_compile' + '_' + str(run) + '_.log' logfile_exec_RRH = logdir_RRH + '/RRH_exec' + '_' + str(run) + '_.log' logfile_pre_exec_RRH = logdir_RRH + '/RRH_pre_exec' + '_' + str(run) + '_.log' logfile_task_RRH_compile_out = logdir_RRH + '/RRH_task_compile_out' + '_' + str(run) + '_.log' logfile_task_RRH_compile = logdir_local_testcase + '/RRH_task_compile' + '_' + str(run) + '_.log' logfile_task_RRH_out = logdir_RRH + '/RRH_task_out' + '_' + str(run) + '_.log' logfile_task_RRH = logdir_local_testcase + '/RRH_task' + '_' + str(run) + '_.log' task_RRH_compile = ' ( uname -a ; date \n' task_RRH_compile = task_RRH_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n' task_RRH_compile = task_RRH_compile + 'env |grep OPENAIR \n' if RRH_compile_prog != "": task_RRH_compile = task_RRH_compile + ' ( ' + RRH_compile_prog + ' '+ RRH_compile_prog_args + ' ) > ' + logfile_compile_RRH + ' 2>&1 \n' task_RRH_compile = task_RRH_compile + ' date ) > ' + logfile_task_RRH_compile_out + ' 2>&1 ' write_file(logfile_task_RRH_compile, task_RRH_compile, mode="w") task_RRH = ' ( uname -a ; date \n' task_RRH = task_RRH + ' export OPENAIR_TESTDIR=' + logdir_RRH + '\n' task_RRH = task_RRH + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n' task_RRH = task_RRH + 'env |grep OPENAIR \n' + 'array_exec_pid=() \n' if RRH_pre_exec != "": task_RRH = task_RRH + ' ( date; ' + RRH_pre_exec + ' '+ RRH_pre_exec_args + ' ) > ' + logfile_pre_exec_RRH + ' 2>&1 \n' if RRH_main_exec != "": task_RRH = task_RRH + ' ( date; ' + addsudo(RRH_main_exec + ' ' + RRH_main_exec_args, mypassword) + ' ) > ' + logfile_exec_RRH + ' 2>&1 & \n' task_RRH = task_RRH + 'array_exec_pid+=($!) \n' task_RRH = task_RRH + 'echo eNB_main_exec PID = $! \n' #terminate the eNB test case after timeout_cmd seconds task_RRH = task_RRH + finalize_deploy_script (timeout_cmd, RRH_terminate_missing_procs) + ' \n' task_RRH = task_RRH + 'handle_ctrl_c' + '\n' task_RRH = task_RRH + ' ) > ' + logfile_task_RRH_out + ' 2>&1 ' write_file(logfile_task_RRH, task_RRH, mode="w") logfile_compile_eNB = logdir_eNB + '/eNB_compile' + '_' + str(run) + '_.log' logfile_exec_eNB = logdir_eNB + '/eNB_exec' + '_' + str(run) + '_.log' logfile_pre_exec_eNB = logdir_eNB + '/eNB_pre_exec' + '_' + str(run) + '_.log' logfile_traffic_eNB = logdir_eNB + '/eNB_traffic' + '_' + str(run) + '_.log' logfile_task_eNB_compile_out = logdir_eNB + '/eNB_task_compile_out' + '_' + str(run) + '_.log' logfile_task_eNB_compile = logdir_local_testcase + '/eNB_task_compile' + '_' + str(run) + '_.log' logfile_task_eNB_out = logdir_eNB + '/eNB_task_out' + '_' + str(run) + '_.log' logfile_task_eNB = logdir_local_testcase + '/eNB_task' + '_' + str(run) + '_.log' logfile_local_traffic_eNB_out = logdir_local_testcase + '/eNB_traffic' + '_' + str(run) + '_.log' logfile_tshark_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.log' logfile_pcap_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.pcap' logfile_pcap_zip_eNB = logdir_eNB + '/eNB_tshark' + '_' + str(run) + '_.pcap.zip' logfile_pcap_tmp_eNB = '/tmp/' + '/eNB_tshark' + '_' + str(run) + '_.pcap' task_eNB_compile = ' ( uname -a ; date \n' task_eNB_compile = task_eNB_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n' task_eNB_compile = task_eNB_compile + 'env |grep OPENAIR \n' task_eNB_compile = task_eNB_compile + update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n' if eNB_compile_prog != "": task_eNB_compile = task_eNB_compile + ' ( ' + eNB_compile_prog + ' '+ eNB_compile_prog_args + ' ) > ' + logfile_compile_eNB + ' 2>&1 \n' task_eNB_compile = task_eNB_compile + ' date ) > ' + logfile_task_eNB_compile_out + ' 2>&1 ' write_file(logfile_task_eNB_compile, task_eNB_compile, mode="w") task_eNB = ' ( uname -a ; date \n' task_eNB = task_eNB + ' export OPENAIR_TESTDIR=' + logdir_eNB + '\n' task_eNB = task_eNB + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n' task_eNB = task_eNB + 'env |grep OPENAIR \n' + 'array_exec_pid=() \n' if eNB_pre_exec != "": task_eNB = task_eNB + ' ( date; ' + eNB_pre_exec + ' '+ eNB_pre_exec_args + ' ) > ' + logfile_pre_exec_eNB + ' 2>&1 \n' if eNB_main_exec != "": task_eNB = task_eNB + ' ( date; ' + addsudo(eNB_main_exec + ' ' + eNB_main_exec_args, mypassword) + ' ) > ' + logfile_exec_eNB + ' 2>&1 & \n' task_eNB = task_eNB + 'array_exec_pid+=($!) \n' task_eNB = task_eNB + 'echo eNB_main_exec PID = $! \n' if eNB_traffic_exec != "": cmd_traffic = eNB_traffic_exec + ' ' + eNB_traffic_exec_args if cmd_traffic.find('-c') >= 0: cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60) task_eNB = task_eNB + ' (date; ' + cmd_traffic + ' ) > ' + logfile_traffic_eNB + ' 2>&1 & \n' task_eNB = task_eNB + 'array_exec_pid+=($!) \n' task_eNB = task_eNB + 'echo eNB_traffic_exec PID = $! \n' task_eNB = task_eNB + ' (date; sudo rm -f ' + logfile_pcap_tmp_eNB + ' ; sudo -E tshark -i lo -s 65535 -a duration:' + str(timeout_cmd-10)+ ' -w ' + logfile_pcap_tmp_eNB+ ' ; sudo -E chown ' + user + ' ' + logfile_pcap_tmp_eNB + ' ; zip -j -9 ' + logfile_pcap_zip_eNB + ' ' + logfile_pcap_tmp_eNB + ' ) > ' + logfile_tshark_eNB + ' 2>&1 & \n ' task_eNB = task_eNB + 'array_exec_pid+=($!) \n' task_eNB = task_eNB + 'echo eNB_tshark_exec PID = $! \n' #terminate the eNB test case after timeout_cmd seconds task_eNB = task_eNB + finalize_deploy_script (timeout_cmd, eNB_terminate_missing_procs) + ' \n' #task_eNB = task_eNB + 'sleep ' + str(timeout_cmd) + ' \n' task_eNB = task_eNB + 'handle_ctrl_c' + '\n' task_eNB = task_eNB + ' ) > ' + logfile_task_eNB_out + ' 2>&1 ' write_file(logfile_task_eNB, task_eNB, mode="w") #task_eNB = 'echo \" ' + task_eNB + '\" > ' + logfile_script_eNB + ' 2>&1 ; ' + task_eNB logfile_compile_UE = logdir_UE + '/UE_compile' + '_' + str(run) + '_.log' logfile_exec_UE = logdir_UE + '/UE_exec' + '_' + str(run) + '_.log' logfile_pre_exec_UE = logdir_UE + '/UE_pre_exec' + '_' + str(run) + '_.log' logfile_traffic_UE = logdir_UE + '/UE_traffic' + '_' + str(run) + '_.log' logfile_task_UE_out = logdir_UE + '/UE_task_out' + '_' + str(run) + '_.log' logfile_task_UE = logdir_local_testcase + '/UE_task' + '_' + str(run) + '_.log' logfile_task_UE_compile_out = logdir_UE + '/UE_task_compile_out' + '_' + str(run) + '_.log' logfile_task_UE_compile = logdir_local_testcase + '/UE_task_compile' + '_' + str(run) + '_.log' logfile_local_traffic_UE_out = logdir_local_testcase + '/UE_traffic' + '_' + str(run) + '_.log' task_UE_compile = ' ( uname -a ; date \n' task_UE_compile = task_UE_compile + 'array_exec_pid=()' + '\n' task_UE_compile = task_UE_compile + 'cd ' + logdirOAI5GRepo + '\n' task_UE_compile = task_UE_compile + 'source oaienv \n' task_UE_compile = task_UE_compile + 'source cmake_targets/tools/build_helper \n' task_UE_compile = task_UE_compile + 'env |grep OPENAIR \n' task_UE_compile = task_UE_compile + update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n' if UE_compile_prog != "": task_UE_compile = task_UE_compile + ' ( ' + UE_compile_prog + ' '+ UE_compile_prog_args + ' ) > ' + logfile_compile_UE + ' 2>&1 \n' task_UE_compile = task_UE_compile + ' ) > ' + logfile_task_UE_compile_out + ' 2>&1 ' write_file(logfile_task_UE_compile, task_UE_compile, mode="w") task_UE = ' ( uname -a ; date \n' task_UE = task_UE + 'array_exec_pid=()' + '\n' task_UE = task_UE + 'cd ' + logdirOAI5GRepo + '\n' task_UE = task_UE + 'source oaienv \n' task_UE = task_UE + 'source cmake_targets/tools/build_helper \n' task_UE = task_UE + 'env |grep OPENAIR \n' if UE_pre_exec != "": task_UE = task_UE + ' ( date; ' + UE_pre_exec + ' '+ UE_pre_exec_args + ' ) > ' + logfile_pre_exec_UE + ' 2>&1 \n' if UE_main_exec != "": task_UE = task_UE + ' ( date; ' + addsudo(UE_main_exec + ' ' + UE_main_exec_args, mypassword) + ' ) > ' + logfile_exec_UE + ' 2>&1 & \n' task_UE = task_UE + 'array_exec_pid+=($!) \n' task_UE = task_UE + 'echo UE_main_exec PID = $! \n' if UE_traffic_exec != "": cmd_traffic = UE_traffic_exec + ' ' + UE_traffic_exec_args if cmd_traffic.find('-c') >= 0: cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60) task_UE = task_UE + ' ( date; ' + cmd_traffic + ' ) >' + logfile_traffic_UE + ' 2>&1 & \n' task_UE = task_UE + 'array_exec_pid+=($!) \n' task_UE = task_UE + 'echo UE_traffic_exec PID = $! \n' #terminate the UE test case after timeout_cmd seconds task_UE = task_UE + finalize_deploy_script (timeout_cmd, UE_terminate_missing_procs) + ' \n' #task_UE = task_UE + 'sleep ' + str(timeout_cmd) + ' \n' task_UE = task_UE + 'handle_ctrl_c' + '\n' task_UE = task_UE + ' ) > ' + logfile_task_UE_out + ' 2>&1 ' write_file(logfile_task_UE, task_UE, mode="w") #task_UE = 'echo \" ' + task_UE + '\" > ' + logfile_script_UE + ' 2>&1 ; ' + task_UE logfile_compile_EPC = logdir_EPC + '/EPC_compile' + '_' + str(run) + '_.log' logfile_compile_HSS = logdir_EPC + '/HSS_compile' + '_' + str(run) + '_.log' logfile_exec_EPC = logdir_EPC + '/EPC_exec' + '_' + str(run) + '_.log' logfile_pre_exec_EPC = logdir_EPC + '/EPC_pre_exec' + '_' + str(run) + '_.log' logfile_exec_HSS = logdir_EPC + '/HSS_exec' + '_' + str(run) + '_.log' logfile_traffic_EPC = logdir_EPC + '/EPC_traffic' + '_' + str(run) + '_.log' logfile_task_EPC_out = logdir_EPC + '/EPC_task_out' + '_' + str(run) + '_.log' logfile_task_EPC = logdir_local_testcase + '/EPC_task' + '_' + str(run) + '_.log' logfile_task_EPC_compile_out = logdir_EPC + '/EPC_task_compile_out' + '_' + str(run) + '_.log' logfile_task_EPC_compile = logdir_local_testcase + '/EPC_task_compile' + '_' + str(run) + '_.log' logfile_local_traffic_EPC_out = logdir_local_testcase + '/EPC_traffic' + '_' + str(run) + '_.log' task_EPC_compile = ' ( uname -a ; date \n' task_EPC_compile = task_EPC_compile + 'array_exec_pid=()' + '\n' task_EPC_compile = task_EPC_compile + 'cd ' + logdirOpenaircnRepo + ' ; source oaienv \n' task_EPC_compile = task_EPC_compile + update_config_file(oai_EPC, EPC_config_file, logdirOpenaircnRepo, logdirOpenaircnRepo+'/TEST/autotests/tools/search_repl.py') + '\n' task_EPC_compile = task_EPC_compile + 'source BUILD/TOOLS/build_helper \n' if EPC_compile_prog != "": task_EPC_compile = task_EPC_compile + '(' + EPC_compile_prog + ' ' + EPC_compile_prog_args + ' ) > ' + logfile_compile_EPC + ' 2>&1 \n' if HSS_compile_prog != "": task_EPC_compile = task_EPC_compile + '(' + HSS_compile_prog + ' ' + HSS_compile_prog_args + ' ) > ' + logfile_compile_HSS + ' 2>&1 \n' task_EPC_compile = task_EPC_compile + ' ) > ' + logfile_task_EPC_compile_out + ' 2>&1 ' write_file(logfile_task_EPC_compile, task_EPC_compile, mode="w") task_EPC = ' ( uname -a ; date \n' task_EPC = task_EPC + ' export OPENAIRCN_TESTDIR=' + logdir_EPC + '\n' task_EPC = task_EPC + 'array_exec_pid=()' + '\n' task_EPC = task_EPC + 'cd ' + logdirOpenaircnRepo + '; source oaienv\n' task_EPC = task_EPC + 'source BUILD/TOOLS/build_helper \n' if EPC_pre_exec != "": task_EPC = task_EPC + ' ( date; ' + EPC_pre_exec + ' '+ EPC_pre_exec_args + ' ) > ' + logfile_pre_exec_EPC + ' 2>&1 \n' if HSS_main_exec != "": task_EPC = task_EPC + '( date; ' + addsudo (HSS_main_exec + ' ' + HSS_main_exec_args, mypassword) + ' ) > ' + logfile_exec_HSS + ' 2>&1 & \n' task_EPC = task_EPC + 'array_exec_pid+=($!) \n' task_EPC = task_EPC + 'echo HSS_main_exec PID = $! \n' if EPC_main_exec != "": task_EPC = task_EPC + '( date; ' + addsudo (EPC_main_exec + ' ' + EPC_main_exec_args, mypassword) + ' ) > ' + logfile_exec_EPC + ' 2>&1 & \n' task_EPC = task_EPC + 'array_exec_pid+=($!) \n' task_EPC = task_EPC + 'echo EPC_main_exec PID = $! \n' if EPC_traffic_exec != "": cmd_traffic = EPC_traffic_exec + ' ' + EPC_traffic_exec_args if cmd_traffic.find('-c') >= 0: cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60) task_EPC = task_EPC + '( date; ' + cmd_traffic + ' ) > ' + logfile_traffic_EPC + ' 2>&1 & \n' task_EPC = task_EPC + 'array_exec_pid+=($!) \n' task_EPC = task_EPC + 'echo EPC_traffic_exec PID = $! \n' #terminate the EPC test case after timeout_cmd seconds task_EPC = task_EPC + finalize_deploy_script (timeout_cmd, EPC_terminate_missing_procs) + '\n' #task_EPC = task_EPC + 'sleep ' + str(timeout_cmd) + '\n' task_EPC = task_EPC + 'handle_ctrl_c' '\n' task_EPC = task_EPC + ' ) > ' + logfile_task_EPC_out + ' 2>&1 ' write_file(logfile_task_EPC, task_EPC, mode="w") #first we compile all the programs but only for run_0 if run == 0: thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC_compile, False, timeout_thread) thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB_compile, False, timeout_thread) thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password , task_UE_compile, False, timeout_thread) if RRHMachine != '': thread_RRH = oaiThread(4, "RRH_thread", RRHMachine, user, password , task_RRH_compile, False, timeout_thread) threads=[] threads.append(thread_eNB) threads.append(thread_UE) threads.append(thread_EPC) if RRHMachine != '': threads.append(thread_RRH) # Start new Threads thread_eNB.start() thread_UE.start() thread_EPC.start() if RRHMachine != '': thread_RRH.start() #Wait for all the compile threads to complete for t in threads: t.join() #Now we execute all the threads thread_EPC = oaiThread(1, "EPC_thread", EPCMachine, user, password , task_EPC, False, timeout_thread) thread_eNB = oaiThread(2, "eNB_thread", eNBMachine, user, password , task_eNB, False, timeout_thread) thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password , task_UE, False, timeout_thread) if RRHMachine != '': thread_RRH = oaiThread(4, "RRH_thread", RRHMachine, user, password , task_RRH, False, timeout_thread) threads=[] threads.append(thread_eNB) threads.append(thread_UE) threads.append(thread_EPC) if RRHMachine != '': threads.append(thread_RRH) # Start new Threads thread_eNB.start() thread_UE.start() thread_EPC.start() if RRHMachine != '': thread_RRH.start() #Wait for all the compile threads to complete for t in threads: t.join() #Now we get the log files from remote machines on the local machine if RRHMachine != '': cleanOldProgramsAllMachines([oai_eNB, oai_UE, oai_EPC, oai_RRH] , oldprogramList, CleanUpAluLteBox, ExmimoRfStop, [logdir_eNB, logdir_UE, logdir_EPC, logdir_RRH], logdirOAI5GRepo) else: cleanOldProgramsAllMachines([oai_eNB, oai_UE, oai_EPC] , oldprogramList, CleanUpAluLteBox, ExmimoRfStop, [logdir_eNB, logdir_UE, logdir_EPC], logdirOAI5GRepo) logfile_UE_stop_script_out = logdir_UE + '/UE_stop_script_out' + '_' + str(run) + '_.log' logfile_UE_stop_script = logdir_local_testcase + '/UE_stop_script' + '_' + str(run) + '_.log' if UE_stop_script != "": cmd = ' ( uname -a ; date \n' cmd = cmd + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n' cmd = cmd + 'env |grep OPENAIR \n' + 'array_exec_pid=() \n' cmd = cmd + UE_stop_script + '\n' cmd = cmd + ') > ' + logfile_UE_stop_script_out + ' 2>&1 ' write_file(logfile_UE_stop_script , cmd, mode="w") thread_UE = oaiThread(4, "UE_thread", UEMachine, user, password , cmd, False, timeout_thread) thread_UE.start() thread_UE.join() #Now we change the permissions of the logfiles to avoid some of them being with root permissions cmd = 'sudo -E chown -R ' + user + ' ' + logdir_eNB res= oai_eNB.send_recv(cmd) print "Changing permissions of logdir <" + logdir_eNB + "> in eNB machine..." + res cmd = 'sudo -E chown -R ' + user + ' ' + logdir_UE res= oai_UE.send_recv(cmd) print "Changing permissions of logdir <" + logdir_UE + "> in UE machine..." + res cmd = 'sudo -E chown -R ' + user + ' ' + logdir_EPC res= oai_EPC.send_recv(cmd) print "Changing permissions of logdir <" + logdir_EPC + "> in EPC machine..." + res if RRHMachine != '': cmd = 'sudo -E chown -R ' + user + ' ' + logdir_RRH res= oai_RRH.send_recv(cmd) print "Changing permissions of logdir <" + logdir_RRH + "> in RRH machine..." + res print "Copying files from EPCMachine : " + EPCMachine + "logdir_EPC = " + logdir_EPC SSHSessionWrapper(EPCMachine, user, None, password, logdir_EPC, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all") print "Copying files from eNBMachine " + eNBMachine + "logdir_eNB = " + logdir_eNB SSHSessionWrapper(eNBMachine, user, None, password, logdir_eNB, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all") print "Copying files from UEMachine : " + UEMachine + "logdir_UE = " + logdir_UE SSHSessionWrapper(UEMachine, user, None, password, logdir_UE, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all") if RRHMachine != '': print "Copying files from RRHMachine : " + RRHMachine + "logdir_RRH = " + logdir_RRH SSHSessionWrapper(RRHMachine, user, None, password, logdir_RRH, logdir_local + '/cmake_targets/autotests/log/'+ testcasename, "get_all") #Currently we only perform throughput tests tput_run_string='' result, tput_string = tput_test_search_expr(eNB_search_expr_true, logfile_local_traffic_eNB_out) tput_run_string = tput_run_string + tput_string run_result=run_result&result result, tput_string = tput_test_search_expr(EPC_search_expr_true, logfile_local_traffic_EPC_out) run_result=run_result&result tput_run_string = tput_run_string + tput_string result, tput_string = tput_test_search_expr(UE_search_expr_true, logfile_local_traffic_UE_out) run_result=run_result&result tput_run_string = tput_run_string + tput_string if run_result == 1: run_result_string = ' RUN_'+str(run) + ' = PASS' else: run_result_string = ' RUN_'+str(run) + ' = FAIL' #If there is assertion, we mark the test case as failure as most likely eNB crashed cmd = "grep -ilr \"assertion\" " + logdir_local_testcase + " | cat " cmd_out = subprocess.check_output ([cmd], shell=True) if len(cmd_out) !=0 : run_result=0 run_result_string = ' RUN_'+str(run) + ' = FAIL(Assert)' #If there is thread busy error, we mark the test case as failure as most likely eNB crashed cmd = "grep -ilr \"thread busy\" " + logdir_local_testcase + " | cat " cmd_out = subprocess.check_output ([cmd], shell=True) if len(cmd_out) !=0: run_result=0 run_result_string = ' RUN_'+str(run) + ' = FAIL(Thread_Busy)' #If there is Segmentation fault, we mark the test case as failure as most likely eNB crashed cmd = "grep -ilr \"segmentation fault\" " + logdir_local_testcase + " | cat " cmd_out = subprocess.check_output ([cmd], shell=True) if len(cmd_out) !=0: run_result=0 run_result_string = ' RUN_'+str(run) + ' = FAIL(SEGFAULT)' run_result_string = run_result_string + tput_run_string test_result=test_result & run_result test_result_string=test_result_string + run_result_string oai_eNB.disconnect() oai_UE.disconnect() oai_EPC.disconnect() #We need to close the new ssh session that was created #if index_eNBMachine == index_EPCMachine: # oai_EPC.disconnect() #Now we finalize the xml file of the test case end_time=time.time() duration= end_time - start_time xmlFile = logdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/test.' + testcasename + '.xml' if test_result ==0: result='FAIL' else: result = 'PASS' xml="\n<testcase classname=\'"+ testcaseclass + "\' name=\'" + testcasename + "."+tags + "\' Run_result=\'" + test_result_string + "\' time=\'" + str(duration) + " s \' RESULT=\'" + result + "\'></testcase> \n" write_file(xmlFile, xml, mode="w") # \brief handler for executing test cases (lte-softmodem-noS1) # \param testcase name of testcase # \param oldprogramList list of programs which must be terminated before running a test case # \param logdirOAI5GRepo directory on remote machine which has openairinterface5g repo installed # \param logdirOpenaircnRepo directory on remote machine which has openair-cn repo installed # \param MachineList list of all machines on which test cases can be run # \param user username with which to login # \param password password with which to login # \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml) # \param ExmimoRfStop command to stop EXMIMO Card # \param nruns_lte-softmodem global parameter to override number of runs (nruns) within the test case def handle_testcaseclass_softmodem_noS1 (testcase, oldprogramList, logdirOAI5GRepo , logdirOpenaircnRepo, MachineList, user, password, CleanUpAluLteBox, ExmimoRfStop, nruns_lte_softmodem, timeout_cmd): #We ignore the password sent to this function for secuirity reasons for password present in log files #It is recommended to add a line in /etc/sudoers that looks something like below. The line below will run sudo without password prompt # your_user_name ALL=(ALL:ALL) NOPASSWD: ALL indent="\t" threads=[] # # Test case parameters # ----------------------------------------------------------------------------- testcase_verdict = 'PASS' testcase_time_start = datetime.datetime.now() testcase_name = testcase.get('id') testcase_class = testcase.findtext('class',default='') testcase_desc = testcase.findtext('desc',default='') if timeout_cmd == '': timeout_cmd = testcase.findtext('TimeOut_cmd',default='') timeout_cmd = int(float(timeout_cmd)) timeout_thread = timeout_cmd + 60 #Timeout_thread is more than that of cmd to have room for compilation time, etc if nruns_lte_softmodem == '': nruns = testcase.findtext('nruns',default='') else: nruns = nruns_lte_softmodem nruns = int(float(nruns)) tags = testcase.findtext('tags',default='') max_ntries = testcase.findtext('max_ntries',default='') if max_ntries : max_ntries = int(float(max_ntries)) else : max_ntries = nruns print( indent + "> testcase time start : " + str(testcase_time_start) ) print( indent + "> testcase class : " + testcase_class ) print( indent + "> testcase description : " + testcase_desc ) print( indent + "> testcase timeout : " + str(timeout_cmd) ) print( indent + "> testcase thread timeout : " + str(timeout_thread) ) print( indent + "> number of runs : " + str(nruns) ) print( indent + "> number of max tries : " + str(max_ntries) ) print( indent + "> testcase tags : " + tags ) logdir_local_testcase = openairdir_local + '/cmake_targets/autotests/log/'+ testcasename logdir_eNB_testcase = logdirOAI5GRepo +'/cmake_targets/autotests/log/'+ testcasename logdir_UE_testcase = logdirOAI5GRepo +'/cmake_targets/autotests/log/'+ testcasename # # Local checks for test cases # ----------------------------------------------- # Logging directory if (not os.path.exists(logdir_local_testcase)): os.system('mkdir -p ' + logdir_local_testcase) # # REMOTE MACHINE COMPILATION # ------------------------------------------------ eNBMachine = testcase.findtext('eNB',default='') eNB_config_file = testcase.findtext('eNB_config_file',default='') eNB_compile_prog = testcase.findtext('eNB_compile_prog',default='') eNB_compile_prog_args = testcase.findtext('eNB_compile_prog_args',default='') logfile_compile_eNB = logdir_eNB_testcase + '/eNB_compile.log' logfile_task_eNB_compile_out = logdir_eNB_testcase + '/eNB_task_compile_out.log' logfile_task_eNB_compile = logdir_local_testcase + '/eNB_task_compile.log' # Check that machine is in test setup machine list print( Fore.WHITE + indent + "> testcase eNB machine :"), if (eNBMachine not in MachineList): print( Fore.RED + eNBMachine + " not in test setup machine list") testcase_verdict = 'INCON' else : print eNBMachine, # get machine description eNBMachineDesc = MachineDescDic[eNBMachine] index_eNBMachine = MachineList.index(eNBMachine) # check that openairinterface is installed on machine oai_eNB = openair('localdomain', eNBMachine) oai_eNB.connect(user, password) cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep --color=never OPENAIR' res= oai_eNB.send_recv(cmd) m = re.search('OPENAIR_HOME', res, re.DOTALL) if m: print # Create testcase directory on remote eNB cmd = 'rm -fr ' + logdir_eNB_testcase + ' ; mkdir -p ' + logdir_eNB_testcase result = oai_eNB.send_recv(cmd) # Check if we need to compile lte-softmodem-noS1 on remote machine eNB_compile_cmd = eNB_compile_prog + ' '+ eNB_compile_prog_args if ('last_compile_prog' in eNBMachineDesc) and eNBMachineDesc['last_compile_prog'] == eNB_compile_cmd: print( Fore.WHITE + indent + "> eNB machine compilation : skipped -> "+eNB_compile_cmd ) else: print( Fore.WHITE + indent + "> eNB machine compilation : triggered -> "+eNB_compile_cmd ) eNBMachineDesc['last_compile_prog'] = eNB_compile_prog + ' '+ eNB_compile_prog_args # if last compilation is the same do not compile again task_eNB_compile = ' ( uname -a ; date \n' task_eNB_compile = task_eNB_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n' task_eNB_compile = task_eNB_compile + 'env |grep OPENAIR \n' task_eNB_compile = task_eNB_compile + update_config_file(oai_eNB, eNB_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n' if eNB_compile_cmd != "": task_eNB_compile = task_eNB_compile + ' ( ' + eNB_compile_cmd + ' ) > ' + logfile_compile_eNB + ' 2>&1 \n' task_eNB_compile = task_eNB_compile + ' date ) > ' + logfile_task_eNB_compile_out + ' 2>&1 ' write_file(logfile_task_eNB_compile, task_eNB_compile, mode="w") thread_eNB = oaiThread(1, "eNB_thread_compile", eNBMachine, user, password , task_eNB_compile, False, timeout_thread) threads.append(thread_eNB) thread_eNB.start() else: print( Fore.RED + " -> OAI5G not properly setup ! please check REMOTE MACHINE PREPARATION step") testcase_verdict = 'INCON' UEMachine = testcase.findtext('UE',default='') UE_config_file = testcase.findtext('UE_config_file',default='') UE_compile_prog = testcase.findtext('UE_compile_prog',default='') UE_compile_prog_args = testcase.findtext('UE_compile_prog_args',default='') logfile_compile_UE = logdir_UE_testcase + '/UE_compile.log' logfile_task_UE_compile_out = logdir_UE_testcase + '/UE_task_compile_out.log' logfile_task_UE_compile = logdir_local_testcase + '/UE_task_compile.log' # Check that machine is in test setup machine list print( Fore.WHITE + indent + "> testcase UE machine :"), if (UEMachine not in MachineList): print( Fore.RED + UEMachine + " not in test setup machine list") testcase_verdict = 'INCON' else : print UEMachine, # get machine description UEMachineDesc = MachineDescDic[UEMachine] index_UEMachine = MachineList.index(UEMachine) # check that openairinterface is installed on machine oai_UE = openair('localdomain', UEMachine) oai_UE.connect(user, password) cmd = 'cd ' + logdirOAI5GRepo + '; source oaienv ; env|grep --color=never OPENAIR' res= oai_UE.send_recv(cmd) m = re.search('OPENAIR_HOME', res, re.DOTALL) if m: print # Create testcase directory on remote UE cmd = 'rm -fr ' + logdir_UE_testcase + ' ; mkdir -p ' + logdir_UE_testcase result = oai_UE.send_recv(cmd) # Check if we need to compile lte-softmodem-noS1 on remote machine UE_compile_cmd = UE_compile_prog + ' '+ UE_compile_prog_args if ('last_compile_prog' in UEMachineDesc) and UEMachineDesc['last_compile_prog'] == UE_compile_cmd: print( Fore.WHITE + indent + "> UE machine compilation : skipped -> "+UE_compile_cmd ) else: print( Fore.WHITE + indent + "> UE machine compilation : triggered -> "+UE_compile_cmd ) UEMachineDesc['last_compile_prog'] = UE_compile_prog + ' '+ UE_compile_prog_args task_UE_compile = ' ( uname -a ; date \n' task_UE_compile = task_UE_compile + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n' task_UE_compile = task_UE_compile + 'env |grep OPENAIR \n' task_UE_compile = task_UE_compile + update_config_file(oai_UE, UE_config_file, logdirOAI5GRepo, '$OPENAIR_DIR/cmake_targets/autotests/tools/search_repl.py') + '\n' if UE_compile_cmd != "": task_UE_compile = task_UE_compile + ' ( ' + UE_compile_cmd + ' ) > ' + logfile_compile_UE + ' 2>&1 \n' task_UE_compile = task_UE_compile + ' date ) > ' + logfile_task_UE_compile_out + ' 2>&1 ' write_file(logfile_task_UE_compile, task_UE_compile, mode="w") thread_UE = oaiThread(2, "UE_thread_compile", UEMachine, user, password , task_UE_compile, False, timeout_thread) threads.append(thread_UE) thread_UE.start() else: print( Fore.RED + " -> OAI5G not properly setup ! please check REMOTE MACHINE PREPARATION step") testcase_verdict = 'INCON' # Wait for Compilation thread to terminate #----------------------------------------- for t in threads: t.join() # TODO check that compilation is succeed #first we compile all the programs # thread_UE = oaiThread(3, "UE_thread", UEMachine, user, password , task_UE_compile, False, timeout_thread) # threads.append(thread_UE) # thread_UE.start() # index_UEMachine = MachineList.index(UEMachine) # oai_UE = openair('localdomain', UEMachine) # oai_UE.connect(user, password) # res = oai_UE.send_recv(cmd) # res = oai_eNB.send_recv(cmd) # # RUN LOOP # ------------------------------------------------ if testcase_verdict != 'PASS': # if something went wrong to not run test cases max_ntries=0 indent="\t\t" runs_results = [] nb_runs = 0 nb_run_pass = 0 nb_run_failed = 0 nb_run_inc = 0 nb_run_skip = 0 nb_seg_fault = 0 for run in range(0,max_ntries): if nruns == nb_run_pass + nb_run_failed: break nb_runs += 1 # # RUN initialization # ---------------------------------------------------- prefix_string = Fore.WHITE + indent + "> RUN_"+str(run).zfill(2)+" :" # print (Fore.WHITE + indent + "> RUN_"+str(run).zfill(2)+" : " ), # sys.stdout.flush() run_start_time=datetime.datetime.now() logdir_local_run = openairdir_local + '/cmake_targets/autotests/log/'+ testcasename + '/run_' + str(run) logdir_eNB_run = logdirOAI5GRepo +'/cmake_targets/autotests/log/' + testcasename + '/run_' + str(run) logdir_UE_run = logdirOAI5GRepo +'/cmake_targets/autotests/log/' + testcasename + '/run_' + str(run) cmd = 'rm -fr ' + logdir_eNB_run + ' ; mkdir -p ' + logdir_eNB_run result = oai_eNB.send_recv(cmd) cmd = 'rm -fr ' + logdir_UE_run + ' ; mkdir -p ' + logdir_UE_run result = oai_UE.send_recv(cmd) cmd = ' rm -fr ' + logdir_local_run + ' ; mkdir -p ' + logdir_local_run result = os.system(cmd) # # RUN parametrization # ---------------------------------------------------- eNB_pre_exec = testcase.findtext('eNB_pre_exec',default='') eNB_pre_exec_args = testcase.findtext('eNB_pre_exec_args',default='') eNB_main_exec = testcase.findtext('eNB_main_exec',default='') eNB_main_exec_args = testcase.findtext('eNB_main_exec_args',default='') eNB_traffic_exec = testcase.findtext('eNB_traffic_exec',default='') eNB_traffic_exec_args = testcase.findtext('eNB_traffic_exec_args',default='') eNB_terminate_missing_procs = testcase.findtext('eNB_terminate_missing_procs',default='True') logfile_exec_eNB = logdir_eNB_run + '/eNB_exec' + '_' + str(run) + '_.log' logfile_pre_exec_eNB = logdir_eNB_run + '/eNB_pre_exec' + '_' + str(run) + '_.log' logfile_task_eNB_out = logdir_eNB_run + '/eNB_task_out' + '_' + str(run) + '_.log' logfile_traffic_eNB = logdir_eNB_run + '/eNB_traffic' + '_' + str(run) + '_.log' logfile_task_eNB = logdir_local_run + '/eNB_task' + '_' + str(run) + '_.log' task_eNB = ' ( uname -a ; date \n' task_eNB = task_eNB + 'cd ' + logdirOAI5GRepo + ' ; source oaienv ; source cmake_targets/tools/build_helper \n' task_eNB = task_eNB + 'env |grep OPENAIR \n' + 'array_exec_pid=() \n' if eNB_pre_exec != "": task_eNB = task_eNB + ' ( date; ' + eNB_pre_exec + ' '+ eNB_pre_exec_args + ' ) > ' + logfile_pre_exec_eNB + ' 2>&1 \n' if eNB_main_exec != "": task_eNB = task_eNB + ' ( date; ' + addsudo(eNB_main_exec + ' ' + eNB_main_exec_args, '') + ' ) > ' + logfile_exec_eNB + ' 2>&1 & \n' task_eNB = task_eNB + 'array_exec_pid+=($!) \n' task_eNB = task_eNB + 'echo eNB_main_exec PID = $! \n' if eNB_traffic_exec != "": cmd_traffic = eNB_traffic_exec + ' ' + eNB_traffic_exec_args if cmd_traffic.find('-c') >= 0: cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 80) task_eNB = task_eNB + ' (date; ' + cmd_traffic + ' ) > ' + logfile_traffic_eNB + ' 2>&1 & \n' task_eNB = task_eNB + 'array_exec_pid+=($!) \n' task_eNB = task_eNB + 'echo eNB_traffic_exec PID = $! \n' #terminate the eNB test case after timeout_cmd seconds task_eNB = task_eNB + finalize_deploy_script (timeout_cmd, eNB_terminate_missing_procs) + ' \n' #task_eNB = task_eNB + 'sleep ' + str(timeout_cmd) + ' \n' task_eNB = task_eNB + 'handle_ctrl_c' + '\n' task_eNB = task_eNB + ' ) > ' + logfile_task_eNB_out + ' 2>&1 ' write_file(logfile_task_eNB, task_eNB, mode="w") UE_pre_exec = testcase.findtext('UE_pre_exec',default='') UE_pre_exec_args = testcase.findtext('UE_pre_exec_args',default='') UE_main_exec = testcase.findtext('UE_main_exec',default='') UE_main_exec_args = testcase.findtext('UE_main_exec_args',default='') UE_traffic_exec = testcase.findtext('UE_traffic_exec',default='') UE_traffic_exec_args = testcase.findtext('UE_traffic_exec_args',default='') UE_terminate_missing_procs = testcase.findtext('UE_terminate_missing_procs',default='True') UE_search_expr_true = testcase.findtext('UE_search_expr_true','') logfile_exec_UE = logdir_UE_run + '/UE_exec' + '_' + str(run) + '_.log' logfile_pre_exec_UE = logdir_UE_run + '/UE_pre_exec' + '_' + str(run) + '_.log' logfile_task_UE_out = logdir_UE_run + '/UE_task_out' + '_' + str(run) + '_.log' logfile_traffic_UE = logdir_UE_run + '/UE_traffic' + '_' + str(run) + '_.log' logfile_task_UE = logdir_local_run + '/UE_task' + '_' + str(run) + '_.log' task_UE = ' ( uname -a ; date \n' task_UE = task_UE + 'array_exec_pid=()' + '\n' task_UE = task_UE + 'cd ' + logdirOAI5GRepo + '\n' task_UE = task_UE + 'source oaienv \n' task_UE = task_UE + 'source cmake_targets/tools/build_helper \n' task_UE = task_UE + 'env |grep OPENAIR \n' if UE_pre_exec != "": task_UE = task_UE + ' ( date; ' + UE_pre_exec + ' '+ UE_pre_exec_args + ' ) > ' + logfile_pre_exec_UE + ' 2>&1 \n' if UE_main_exec != "": task_UE = task_UE + ' ( date; ' + addsudo(UE_main_exec + ' ' + UE_main_exec_args, '') + ' ) > ' + logfile_exec_UE + ' 2>&1 & \n' task_UE = task_UE + 'array_exec_pid+=($!) \n' task_UE = task_UE + 'echo UE_main_exec PID = $! \n' if UE_traffic_exec != "": cmd_traffic = UE_traffic_exec + ' ' + UE_traffic_exec_args if cmd_traffic.find('-c') >= 0: cmd_traffic = cmd_traffic + ' -t ' + str(timeout_cmd - 60) task_UE = task_UE + ' ( date; ' + cmd_traffic + ' ) >' + logfile_traffic_UE + ' 2>&1 & \n' task_UE = task_UE + 'array_exec_pid+=($!) \n' task_UE = task_UE + 'echo UE_traffic_exec PID = $! \n' #terminate the UE test case after timeout_cmd seconds task_UE = task_UE + finalize_deploy_script (timeout_cmd, UE_terminate_missing_procs) + ' \n' #task_UE = task_UE + 'sleep ' + str(timeout_cmd) + ' \n' task_UE = task_UE + 'handle_ctrl_c' + '\n' task_UE = task_UE + ' ) > ' + logfile_task_UE_out + ' 2>&1 ' write_file(logfile_task_UE, task_UE, mode="w") #task_UE = 'echo \" ' + task_UE + '\" > ' + logfile_script_UE + ' 2>&1 ; ' + task_UE # # RUN work # ---------------------------------------------------- thread_eNB = oaiThread(1, "eNB_thread", eNBMachine, user, password , task_eNB, False, timeout_thread) thread_UE = oaiThread(2, "UE_thread", UEMachine, user, password , task_UE, False, timeout_thread) threads=[] threads.append(thread_eNB) threads.append(thread_UE) thread_eNB.start() thread_UE.start() run_work_inprogress_flag = True progress_step = timeout_thread/20 progress_count = 0 while (run_work_inprogress_flag): update_progress(float(progress_count)/float(timeout_thread),prefix_string) time.sleep(progress_step) progress_count += progress_step # sys.stdout.write ('#') # sys.stdout.flush() run_work_inprogress_flag = False for t in threads: if t.isAlive(): run_work_inprogress_flag = True update_progress(1,prefix_string) for t in threads: t.join() # # #----------------------------------------------------- cleanOldProgramsAllMachines([oai_eNB, oai_UE] , oldprogramList, CleanUpAluLteBox, ExmimoRfStop, [logdir_eNB_run, logdir_UE_run], logdirOAI5GRepo) SSHSessionWrapper(eNBMachine, user, None, password, logdir_eNB_run, logdir_local_testcase, "get_all") #print "Copying files from UEMachine : " + UEMachine + "logdir_UE = " + logdir_UE SSHSessionWrapper(UEMachine, user, None, password, logdir_UE_run, logdir_local_testcase, "get_all") metric_checks_flag = 1 fname = logdir_local_run+ '/UE_exec' + '_' + str(run) + '_.log' cell_synch_status = analyser.check_cell_synchro(fname) print (Fore.WHITE + indent + "> Check Cell synchro :"), if cell_synch_status == 'CELL_SYNCH': print ( Fore.GREEN + cell_synch_status) else : print ( Fore.RED + cell_synch_status) metric_checks_flag = 0 ue_seg_fault_status = analyser.check_exec_seg_fault(fname) if ue_seg_fault_status == 'SEG_FAULT' : nb_seg_fault += 1 # # Check metrics #---------------------------------------------------- runs_metrics = [] run_traffic = [] if metric_checks_flag : verdict = 'PASS' # UE side metrics metricList=testcase.findall('UE_metric') for metric in metricList: metric_verdict = 'PASS' metric_def = {} metric_def['id'] = metric.get('id') metric_def['description'] = metric.get('description') metric_def['regex'] = metric.get('regex') metric_def['unit_of_meas'] = metric.get('unit_of_meas') metric_def['pass_fail_stat']= metric.get('pass_fail_stat') metric_min_lim = metric.get('min_limit') if metric_min_lim: metric_def['min_limit'] = float(metric_min_lim) metric_max_lim = metric.get('max_limit') if metric_max_lim: metric_def['max_limit'] = float(metric_max_lim) fname = logdir_local_run+ '/UE_exec' + '_' + str(run) + '_.log' args = {'metric' : metric_def, 'file' : fname } metric_extracted = analyser.do_extract_metrics(args) print (Fore.WHITE + indent + "> Metric : "+metric_def['id']), print ("(min="+str( metric_extracted['metric_min'])+", max="+str( metric_extracted['metric_max'])+", mean="+str( metric_extracted['metric_mean'])+", median="+str( metric_extracted['metric_median'])+")") # print "\t > Metric "+metric_def['id']+" :" # print "\t\t> min = "+ str( metric_extracted['metric_min'] ) # print "\t\t> min_index = "+ str( metric_extracted['metric_min_index'] ) # print "\t\t> max = "+ str( metric_extracted['metric_max'] ) # print "\t\t> max_index = "+ str( metric_extracted['metric_max_index'] ) # print "\t\t> mean = "+ str( metric_extracted['metric_mean'] ) # print "\t\t> median = "+ str( metric_extracted['metric_median'] ) if metric_def['pass_fail_stat'] : metric_verdict = analyser.do_check_verdict(metric_def, metric_extracted) metric_fig = logdir_local_run+ '/UE_metric_'+ metric_def['id']+'_' + str(run) + '_.png' analyser.do_img_metrics(metric_def, metric_extracted, metric_fig) metric_fig_report = '../log/'+ testcasename + '/run_' + str(run) + '/UE_metric_'+ metric_def['id']+'_' + str(run) + '_.png' run_metrics = dict( metric_id = metric_def['id'], metric_desc = metric_def['description'], metric_uom = metric_def['unit_of_meas'], metric_min = metric_extracted['metric_min'], metric_min_index= metric_extracted['metric_min_index'], metric_max = metric_extracted['metric_max'], metric_max_index= metric_extracted['metric_max_index'], metric_mean = metric_extracted['metric_mean'], metric_median = metric_extracted['metric_median'], metric_fig = metric_fig_report) if metric_def['pass_fail_stat'] : run_metrics['pass_fail_stat'] = metric_def['pass_fail_stat'] if metric_min_lim : run_metrics['pass_fail_min_limit'] = metric_def['min_limit'] if metric_max_lim : run_metrics['pass_fail_max_limit'] = metric_def['max_limit'] runs_metrics.append(run_metrics) if metric_verdict != 'PASS': verdict = metric_verdict # End Metrics LOOP # --------------------- # Traffic analysis if UE_traffic_exec != "": fname = logdir_local_run+ '/UE_traffic' + '_' + str(run) + '_.log' args = {'file' : fname } traffic_metrics = analyser.do_extract_traffic_metrics(args) traffic_fig = logdir_local_run+ '/UE_traffic'+'_' + str(run) + '_.png' analyser.do_img_traffic(traffic_metrics, traffic_fig) traffic_fig_report= '../log/'+ testcasename + '/run_' + str(run) + '/UE_traffic'+'_' + str(run) + '_.png' dur_pass_fail_crit = 'none' if traffic_metrics['traffic_count'] == 0 : verdict = 'FAIL' elif UE_search_expr_true != "": traffic_duration_limit = float(UE_search_expr_true) dur_pass_fail_crit = str(traffic_duration_limit) # print 'traffic_duration_limit = '+str(traffic_duration_limit) # print 'traffic_metrics[interval_stop_max] = '+str(traffic_metrics['interval_stop_max']) if traffic_metrics['interval_stop_max'] < traffic_duration_limit : verdict = 'FAIL' run_traffic = dict( traffic_count = traffic_metrics['traffic_count'], bw_min = traffic_metrics['bw_min'], bw_max = traffic_metrics['bw_max'], bw_mean = traffic_metrics['bw_mean'], bw_median = traffic_metrics['bw_median'], jitter_min = traffic_metrics['jitter_min'], jitter_max = traffic_metrics['jitter_max'], jitter_mean = traffic_metrics['jitter_mean'], jitter_median = traffic_metrics['jitter_median'], rl_min = traffic_metrics['rl_min'], rl_max = traffic_metrics['rl_max'], rl_mean = traffic_metrics['rl_mean'], rl_median = traffic_metrics['rl_median'], iperf_duration = traffic_metrics['interval_stop_max'], dur_pass_fail_crit = dur_pass_fail_crit, traffic_fig = traffic_fig_report ) else: run_traffic = dict( traffic_count = 0) else : verdict = 'SKIP' # # RUN verdict and finalization # ---------------------------------------------------- run_stop_time=datetime.datetime.now() run_duration = run_stop_time-run_start_time # print (Fore.WHITE + ("duration=" :"), print (Fore.WHITE + indent + "> RUN duration : "+ str(run_duration) +"s" ) print (Fore.WHITE + indent + "> RUN verdict :"), if verdict == 'PASS': nb_run_pass += 1 print ( Fore.GREEN + verdict) elif verdict == 'FAIL': nb_run_failed += 1 testcase_verdict = 'FAIL' print ( Fore.RED + verdict) elif verdict == 'SKIP': nb_run_skip += 1 print ( Fore.YELLOW + verdict) else: nb_run_inc += 1 if testcase_verdict == 'PASS' : testcase_verdict = 'INCON' print (Fore.YELLOW+'INCONCLUSIVE') run_results = dict( run_id=str(run), run_start_time = run_start_time, run_stop_time = run_stop_time , run_verdict = verdict, ue_seg_fault_status = ue_seg_fault_status, run_duration = run_duration, runs_metrics = runs_metrics, run_traffic = run_traffic) runs_results.append(run_results) # END RUN LOOP #---------------------------------------------------- indent="\t" # Test case duration # ---------------------------------- testcase_time_stop = datetime.datetime.now() print(Fore.WHITE + indent + "> testcase time stop : " + str(testcase_time_start) ), print " -> TestCase duration = "+str(testcase_time_stop - testcase_time_start) # Save remote log files # ----------------------------------- #Now we change the permissions of the logfiles to avoid some of them being with root permissions cmd = 'sudo -E chown -R ' + user + ' ' + logdir_eNB_testcase res= oai_eNB.send_recv(cmd) cmd = 'sudo -E chown -R ' + user + ' ' + logdir_UE_testcase res= oai_UE.send_recv(cmd) # Save remote log files on MTC #print "Copying files from eNBMachine " + eNBMachine + "logdir_eNB = " + logdir_eNB SSHSessionWrapper(eNBMachine, user, None, password, logdir_eNB_testcase, openairdir_local + '/cmake_targets/autotests/log/', "get_all") #print "Copying files from UEMachine : " + UEMachine + "logdir_UE = " + logdir_UE SSHSessionWrapper(UEMachine, user, None, password, logdir_UE_testcase, openairdir_local + '/cmake_targets/autotests/log/', "get_all") oai_eNB.disconnect() oai_UE.disconnect() # Set test case final verdict # ----------------------------------- test_result_string = 'do be completed' if testcase_verdict == 'PASS': if nb_run_pass == 0 : testcase_verdict = 'INCON' print(Fore.WHITE + indent + "> testcase final verdict :"), if testcase_verdict == 'PASS': print ( Fore.GREEN + testcase_verdict) elif testcase_verdict == 'FAIL': print ( Fore.RED + testcase_verdict) else: print (Fore.YELLOW+'INCONCLUSIVE') print(Style.RESET_ALL) duration= testcase_time_stop - testcase_time_start test_result = dict(testcase_name=testcasename, testcaseclass=testcaseclass, testcase_verdict = testcase_verdict, testcase_time_start=testcase_time_start, testcase_time_stop=testcase_time_stop, tags=tags, nruns=nb_runs, nb_run_pass = nb_run_pass, nb_run_skip = nb_run_skip, nb_run_failed = nb_run_failed, nb_run_inc=nb_run_inc, nb_seg_fault = nb_seg_fault, testcase_timeout=timeout_cmd, testcase_duration = duration, testcase_eNBMachine =eNBMachine, testcase_UEMachine =UEMachine, runs_results = runs_results) test_results.append(test_result) xmlFile = logdir_local_testcase + '/test.' + testcasename + '.xml' xml="\n<testcase classname=\'"+ testcaseclass + "\' name=\'" + testcasename + "."+tags + "\' Run_result=\'" + test_result_string + "\' time=\'" + str(duration) + " s \' RESULT=\'" + testcase_verdict + "\'></testcase> \n" write_file(xmlFile, xml, mode="w") # xmlFile_ng = logdir_local_testcase + '/test.' + testcasename + '_ng.xml' # xml_ng = xmlify(test_result, wrap=testcasename, indent=" ") # write_file(xmlFile_ng, xml_ng, mode="w") return testcase_verdict # \brief This function searches if test case is present in list of test cases that need to be executed by user # \param testcasename the test case to search for # \param testcasegroup list that is passed from the arguments # \param test_case_exclude list of test cases excluded from execution (specified in test_case_list.xml) def search_test_case_group(testcasename, testcasegroup, test_case_exclude): if test_case_exclude != "": testcase_exclusion_list=test_case_exclude.split() for entry in testcase_exclusion_list: if entry.find('+') >=0: match = re.search(entry, testcasename) if match: #print "\nSkipping test case as it is found in black list: " + testcasename return False else: match = entry.find(testcasename) if match >=0: #print "\nSkipping test case as it is found in black list: " + testcasename return False if testcasegroup == '': return True else: testcaselist = testcasegroup.split() for entry in testcaselist: if entry.find('+') >=0: match = re.search(entry, testcasename) if match: return True else: match = testcasename.find(entry) if match >=0: return True return False # \brief thread that cleans up remote machines from pre-existing test case executions # \param threadID number of thread (for book keeping) # \param threadname name of thread (for book keeping) # \param oai handler that can be used to execute programs on remote machines # \param CleanUpOldProgs list of programs which must be terminated before running a test case (specified in test_case_list.xml) # \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml) # \param ExmimoRfStop command to stop EXMIMO Card class oaiCleanOldProgramThread (threading.Thread): def __init__(self, threadID, threadname, oai, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop, logdir, logdirOAI5GRepo): threading.Thread.__init__(self) self.threadID = threadID self.threadname = threadname self.oai = oai self.CleanUpOldProgs = CleanUpOldProgs self.CleanUpAluLteBox = CleanUpAluLteBox self.ExmimoRfStop = ExmimoRfStop self.logdir = logdir self.logdirOAI5GRepo = logdirOAI5GRepo def run(self): try: cleanOldPrograms(self.oai, self.CleanUpOldProgs, self.CleanUpAluLteBox, self.ExmimoRfStop, self.logdir, self.logdirOAI5GRepo) except Exception, e: error='' error = error + ' In class oaiCleanOldProgramThread, function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e) error = error + '\n threadID = ' + str(self.threadID) + '\n threadname = ' + self.threadname + '\n CleanUpOldProgs = ' + self.CleanUpOldProgs + '\n CleanUpAluLteBox = ' + self.CleanUpAluLteBox + '\n ExmimoRfStop = ' + self.ExmimoRfStop + '\n' error = error + traceback.format_exc() print error print "There is error in cleaning up old programs....." #sys.exit(1) # \brief Run parallel threads in all machines for clean up old execution of test cases # \param oai_list list of handlers that can be used to execute programs on remote machines # \param CleanUpOldProgs list of programs which must be terminated before running a test case (specified in test_case_list.xml) # \param CleanupAluLteBox string that contains commands to stop ALU Bell Labs LTEBox (specified in test_case_list.xml) # \param ExmimoRfStop command to stop EXMIMO Card def cleanOldProgramsAllMachines(oai_list, CleanOldProgs, CleanUpAluLteBox, ExmimoRfStop, logdir_list, logdirOAI5GRepo): threadId=0 threadList=[] index=0 if len(oai_list)!=len(logdir_list) : logdir_list=[logdir[0]]*len(oai_list) for oai in oai_list: threadName="cleanup_thread_"+str(threadId) thread=oaiCleanOldProgramThread(threadId, threadName, oai, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop, logdir_list[index],logdirOAI5GRepo) threadList.append(thread) thread.start() threadId = threadId + 1 index = index+1 for t in threadList: t.join() debug = 0 pw ='' i = 0 dlsim=0 localshell=0 GitOAI5GRepo='' GitOAI5GRepoBranch='' GitOAI5GHeadVersion='' user='' pw='' testcasegroup='' NFSResultsShare='' cleanUpRemoteMachines=False openairdir_local = os.environ.get('OPENAIR_DIR') if openairdir_local is None: print "Environment variable OPENAIR_DIR not set correctly" sys.exit() locallogdir = openairdir_local + '/cmake_targets/autotests/log' reportdir = openairdir_local + '/cmake_targets/autotests/report' MachineList = '' MachineListGeneric='' MachineDescDic={} flag_remove_logdir=False flag_start_testcase=False nruns_lte_softmodem='' flag_skip_git_head_check=False flag_skip_oai_install=False flag_skip_machine_preparation=False flag_skip_sanity_check=False flag_generate_html_report = False Timeout_cmd='' xmlInputFile='' #print "Number of arguments argc = " + str(len(sys.argv)) #for index in range(1,len(sys.argv) ): # print "argv_" + str(index) + " : " + sys.argv[index] oai_list = [] i=1 while i < len (sys.argv): arg=sys.argv[i] if arg == '-r': flag_remove_logdir=True elif arg == '-s' : flag_start_testcase=True elif arg == '-g' : testcasegroup = sys.argv[i+1].replace("\"","") i = i +1 elif arg == '-c': cleanUpRemoteMachines=True elif arg == '-5GRepo': GitOAI5GRepo = sys.argv[i+1] i = i +1 elif arg == '-5GRepoBranch': GitOAI5GRepoBranch = sys.argv[i+1] i = i +1 elif arg == '-5GRepoHeadVersion': GitOAI5GHeadVersion = sys.argv[i+1] #We now find the branch that corresponds to this Git Head Commit cmd = "git show-ref --head " + " | grep " + GitOAI5GHeadVersion cmd_out = subprocess.check_output ([cmd], shell=True) cmd_out=cmd_out.replace("\n","") cmd_out = cmd_out.split('/') GitOAI5GRepoBranch = cmd_out[-1] if GitOAI5GRepoBranch == '': print "Error extracting GitBranch from head commit. Exiting now..." sys.exit(1) i = i +1 elif arg == '-u': user = sys.argv[i+1] i = i +1 elif arg == '-p': pw = sys.argv[i+1] i = i +1 elif arg == '-n': NFSResultsShare = sys.argv[i+1] i = i +1 elif arg == '--nrun_lte_softmodem': nruns_lte_softmodem = sys.argv[i+1] i = i +1 elif arg == '-MachineList': MachineList = sys.argv[i+1] MachineList = MachineList.replace("\"","") MachineList = MachineList.replace("\'","") i = i +1 elif arg == '-MachineListGeneric': MachineListGeneric = sys.argv[i+1] MachineListGeneric = MachineListGeneric.replace("\"","") MachineListGeneric = MachineListGeneric.replace("\'","") i = i +1 elif arg == '--skip-git-head-check': flag_skip_git_head_check=True elif arg == '--timeout_cmd': Timeout_cmd = sys.argv[i+1] i = i +1 elif arg == '--skip-oai-install': flag_skip_oai_install=True elif arg == '--skip-machine-preparation': flag_skip_machine_preparation=True elif arg == '--skip-sanity-check': flag_skip_sanity_check=True elif arg == '--test-suite' : xmlInputFile = sys.argv[i+1] i = i +1 elif arg == '--generate-html-report': flag_generate_html_report=True elif arg == '-h' : print "-s: This flag *MUST* be set to start the test cases" print "-r: Remove the log directory in autotests" print "-g: Run test cases in a group" print "-c: Run cleanup scripts on remote machines and exit" print "-5GRepo: Repository for OAI 5G to use to run tests (overrides GitOAI5GRepo in test_case_list.xml)" print "-5GRepoBranch: Branch for OAI 5G Repository to run tests (overrides the branch in test_case_list.xml)" print "-5GRepoHeadVersion: Head commit on which to run tests (overrides the branch in test_case_list.xml)" print "-u: use the user name passed as argument" print "-p: use the password passed as an argument" print "-n: Set the NFS share passed as an argument" print "--nrun_lte_softmodem: Set the number of runs for lte-softmodem test case class" print "-MachineList : overrides the MachineList parameter in test_case_list.xml" print "-MachineListGeneric : overrides the MachineListGeneric parameter in test_case_list.xml" print "--skip-git-head-check: skip checking of GitHead remote/local branch (only for debugging)" print "--timeout_cmd: Override the default parameter (timeout_cmd) in test_case_list.xml. This parameter is in seconds and should be > 120" print "--skip-oai-install: Skips the openairinterface5g installer" print "--skip-machine-preparation: skipp the whole system preparation -> direct to test cases" print "--skip-sanity-check: skipp the machine sanity checks" print "--test-suite: Select a XML test-suite file" print "--generate-html-report: Generate an HTML report of the test campaign (supported only for noS1 testsuite !!)" sys.exit() else : print "Unrecongnized Option: <" + arg + ">. Use -h to see valid options" sys.exit() i= i + 1 logdir = '/tmp/' + 'OAITestFrameWork-' + user + '/' logdirOAI5GRepo = logdir + 'openairinterface5g/' logdirOpenaircnRepo = logdir + 'openair-cn/' patchdir = logdirOAI5GRepo + 'cmake_targets/autotests/patches/' NFSResultsDir = '/mnt/sradio' cleanupOldProgramsScript = '$OPENAIR_DIR/cmake_targets/autotests/tools/remove_old_programs.bash' #Now we parse the xml file for basic configuration if xmlInputFile == '': xmlInputFile=os.environ.get('OPENAIR_DIR')+"/cmake_targets/autotests/test_case_list.xml" xmlTree = ET.parse(xmlInputFile) xmlRoot = xmlTree.getroot() # # Read test session configuration # --------------------------------------------------------------------------------- print (Fore.YELLOW + '\nStep 0.1 - Read test session configuration') print (Fore.YELLOW + '-------------------------------------------------') test_session_start_time = datetime.datetime.now() print (Fore.WHITE + ' - start time : '+ str(test_session_start_time)) host = os.uname()[1] print (Fore.WHITE + ' - MTC host : '+host) if user=='': user = getpass.getuser() print (Fore.WHITE + ' - user : '+user) if pw=='': pw = getpass.getpass() print (Fore.WHITE + ' - password : '+pw) print (Fore.WHITE + ' - cmd line flags : ') print (Fore.WHITE + ' + -s : '+str(flag_start_testcase)) print (Fore.WHITE + ' + -r : '+str(flag_remove_logdir)) print (Fore.WHITE + ' + -c : '+str(cleanUpRemoteMachines)) print (Fore.WHITE + ' + --skip-git-head-check : '+str(flag_skip_git_head_check)) print (Fore.WHITE + ' + --skip-oai-install : '+str(flag_skip_oai_install)) print (Fore.WHITE + ' + --skip-machine-preparation : '+str(flag_skip_machine_preparation)) print (Fore.WHITE + ' + --skip-sanity-check : '+str(flag_skip_sanity_check)) print (Fore.WHITE + ' - Test setup basic config : (* -> overwrite by cmd line parameter)') print (Fore.WHITE + ' + test setup config file : '+str(xmlInputFile)) print (Fore.WHITE + ' + remote setup dir : '+str(logdir)) print (Fore.WHITE + ' + remote OAI5G repo : '+str(logdirOAI5GRepo)) print (Fore.WHITE + ' + remote OpenAirCN repo : '+str(logdirOpenaircnRepo)) if MachineList =='': MachineList = xmlRoot.findtext('MachineList',default='') NFSResultsShare = xmlRoot.findtext('NFSResultsShare',default='') GitOpenaircnRepo = xmlRoot.findtext('GitOpenair-cnRepo',default='') if GitOAI5GRepo == '': GitOAI5GRepo = xmlRoot.findtext('GitOAI5GRepo',default='') if GitOAI5GRepoBranch == '': GitOAI5GRepoBranch = xmlRoot.findtext('GitOAI5GRepoBranch',default='') if GitOAI5GHeadVersion == '': GitOAI5GHeadVersion = xmlRoot.findtext('GitOAI5GHeadVersion',default='') GitOpenaircnRepoBranch = xmlRoot.findtext('GitOpenair-cnRepoBranch',default='') CleanUpOldProgs = xmlRoot.findtext('CleanUpOldProgs',default='') CleanUpAluLteBox = xmlRoot.findtext('CleanUpAluLteBox',default='') Timeout_execution = int (xmlRoot.findtext('Timeout_execution')) if MachineListGeneric == '': MachineListGeneric = xmlRoot.findtext('MachineListGeneric',default='') TestCaseExclusionList = xmlRoot.findtext('TestCaseExclusionList',default='') ExmimoRfStop = xmlRoot.findtext('ExmimoRfStop',default='') if nruns_lte_softmodem == '': nruns_lte_softmodem = xmlRoot.findtext('nruns_lte-softmodem',default='') print (Fore.WHITE + ' + MachineList : '+ MachineList) print (Fore.WHITE + ' + GitOpenair-cnRepo : '+ GitOpenaircnRepo) print (Fore.WHITE + ' + GitOAI5GRepo : '+ GitOAI5GRepo) print (Fore.WHITE + ' + GitOAI5GBranch : '+ GitOAI5GRepoBranch) print (Fore.WHITE + ' + GitOpenaircnRepoBranch : '+ GitOpenaircnRepoBranch) print (Fore.WHITE + ' + NFSResultsShare : '+ NFSResultsShare) print (Fore.WHITE + ' + nruns_lte_softmodem : '+ nruns_lte_softmodem) print (Fore.WHITE + ' + Timeout_cmd : '+ Timeout_cmd) if GitOAI5GHeadVersion == '': cmd = "git log --pretty=format:\'%H\' -n 1" # origin/"+ GitOAI5GRepoBranch # cmd = "git show-ref --heads -s "+ GitOAI5GRepoBranch GitOAI5GHeadVersion = subprocess.check_output ([cmd], shell=True) GitOAI5GHeadVersion=GitOAI5GHeadVersion.replace("\n","") # Print commiter and date : git log -1 --pretty=format:"%an %ad" print (Fore.WHITE + ' + GitOAI5GHeadVersion : '+ GitOAI5GHeadVersion) #print "CleanUpOldProgs = " + CleanUpOldProgs print (Fore.WHITE + ' + Timeout_execution (s) : '+ str(Timeout_execution)) if GitOAI5GHeadVersion == '': print "Error getting the OAI5GBranch Head version...Exiting" sys.exit() NFSTestsResultsDir = NFSResultsShare + '/'+ GitOAI5GRepoBranch + '/' + GitOAI5GHeadVersion #print "NFSTestsResultsDir = " + NFSTestsResultsDir OAI5GpatchFileList=xmlRoot.findall('OAI5GPatchFile') MachineList = MachineList.split() MachineListGeneric = MachineListGeneric.split() MachineDescDic # # 'Check Mater Test Controller sanity (MTC)' # --------------------------------------------------------------------------------- print (Fore.YELLOW + '\nStep 0.2 - Check Mater Test Controller sanity (MTC)') print (Fore.YELLOW + '-------------------------------------------------') try: print (Fore.WHITE + ' - check OpenAirInterface environment variable OPENAIR1_DIR... '), os.environ["OPENAIR1_DIR"] except KeyError: print (Fore.RED + 'FAILED') print "Please set the environment variable OPENAIR1_DIR in the .bashrc or run 'source oaienv' " exit_prog(1) print (Fore.GREEN + 'OK') try: print (Fore.WHITE + ' - check OpenAirInterface environment variable OPENAIR2_DIR... '), os.environ["OPENAIR2_DIR"] except KeyError: print (Fore.RED + 'FAILED') print "Please set the environment variable OPENAIR2_DIR in the .bashrc or run 'source oaienv' " exit_prog(1) print (Fore.GREEN + 'OK') try: os.environ["OPENAIR_TARGETS"] except KeyError: print "Please set the environment variable OPENAIR_TARGETS in the .bashrc or run 'source oaienv' " sexit_prog(1) print (Fore.WHITE + ' - kill zombie ssh sessions from earlier sessions... '), cmd='ps aux |grep \"/usr/bin/ssh -q -l ' + user + '\"| tr -s \" \" :| cut -f 2 -d :| xargs kill -9 ' os.system(cmd) print (Fore.GREEN + 'DONE') if flag_start_testcase == False: print "You need to start the testcase by passing option -s. Use -h to see all options. Aborting now..." exit_prog(1) #print "Killing zombie ssh sessions from earlier sessions..." #cmd='ps aux |grep \"/usr/bin/ssh -q -l guptar\"|tr -s \" \" :|cut -f 2 -d :|xargs kill -9 ' #cmd = cmd + '; ps aux |grep \"/usr/bin/ssh -q -l ' + user + '\"| tr -s \" \" :| cut -f 2 -d :| xargs kill -9 ' #os.system(cmd) #print cmd try: analyser = __import__("lib_autotest_analyser") except ImportError as err: print('Import error: ' + str(err)) exit(0) if flag_remove_logdir == True: print "Removing directory: " + locallogdir os.system(' rm -fr ' + locallogdir + '; mkdir -p ' + locallogdir ) if not os.path.exists(locallogdir): os.system('mkdir -p ' + locallogdir ) paramiko_logfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/log/paramiko.log') res=os.system(' echo > ' + paramiko_logfile) paramiko.util.log_to_file(paramiko_logfile) # # 'REMOTE MACHINE SANITY CHEKS' # --------------------------------------------------------------------------------- print (Fore.YELLOW + '\nStep 0.3 - REMOTE MACHINE SANITY CHEKS') print (Fore.YELLOW + '-------------------------------------------------') for machine in MachineList: oai_list.append( openair('localdomain',machine)) MachineDescDic[machine]={} if not flag_skip_sanity_check: if localshell == 0: try: index=0 for machine in MachineList: print (Fore.WHITE + " - Check sanity for remote machine "+machine) oai_list[index].connect(user,pw) print (Fore.WHITE + " > Checking for sudo permissions for user "+user+ " : "), result = oai_list[index].send_expect_false('sudo -S -v','may not run sudo',True) if (result == 'OK'): print (Fore.GREEN + result ) else: print (Fore.RED + result ) print (Fore.WHITE + " > Cleaning Older running programs : "), cleanOldPrograms(oai_list[index], CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop, '$HOME', '/tmp') print (Fore.WHITE + " Done") #result = oai_list[index].send('mount ' + NFSResultsDir, True) #print "Mounting NFS Share " + NFSResultsDir + "..." + result # Check if NFS share is mounted correctly. #print 'Checking if NFS Share<' + NFSResultsDir + '> is mounted correctly...' #cmd = 'if grep -qs '+NFSResultsDir+ ' /proc/mounts; then echo \'' + NFSResultsDir + ' is mounted\' ; fi' #search_expr = NFSResultsDir + ' is mounted' #print "cmd = " + cmd #print "search_expr = " + search_expr #result = oai_list[index].send_expect(cmd, search_expr) #print "Mount NFS_Results_Dir..." + result index = index + 1 #oai.connect2(user,pw) #oai.get_shell() except : print 'Fail to connect to the machine: '+ machine sys.exit(1) else: pw = '' oai_list[0].connect_localshell() else: print (Fore.RED + " Skipping Machine sanity check...") # # 'REMOTE MACHINE TEST SETUP CONFIGURATION' # --------------------------------------------------------------------------------- print (Fore.YELLOW + '\nStep 0.4 - REMOTE MACHINE PREPARATION') print (Fore.YELLOW + '-------------------------------------------------') if not flag_skip_machine_preparation: index=0 threads_init_setup=[] for oai in oai_list: try: print "setting up machine: " + MachineList[index] #print oai_list[oai].send_recv('echo \''+pw+'\' |sudo -S -v') #print oai_list[oai].send_recv('sudo su') #print oai_list[oai].send_recv('who am i') #cleanUpPrograms(oai_list[oai] cmd = 'sudo -S -E rm -fr ' + logdir + ' ; mkdir -p ' + logdir result = oai.send_recv(cmd) print cmd setuplogfile = logdir + '/setup_log_' + MachineList[index] + '_.txt' setup_script = locallogdir + '/setup_script_' + MachineList[index] + '_.txt' #Sometimes git fails so the script below retries in that case localfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/tools/git-retry.sh') remotefile = logdir + '/git-retry.sh' paramList=[] port=22 paramList.append ( {"operation":'put', "localfile":localfile, "remotefile":remotefile} ) sftp_log = os.path.expandvars(locallogdir + '/sftp_module.log') sftp_module (user, pw, MachineList[index], port, paramList, sftp_log) cmd = ' ( \n' #cmd = cmd + 'rm -fR ' + logdir + '\n' #cmd = cmd + 'mkdir -p ' + logdir + '\n' cmd = cmd + 'cd '+ logdir + '\n' cmd = cmd + 'sudo apt-get install -y git \n' cmd = cmd + 'chmod 700 ' + logdir + '/git-retry.sh \n' cmd = cmd + logdir + '/git-retry.sh clone '+ GitOAI5GRepo +' \n' cmd = cmd + logdir + '/git-retry.sh clone '+ GitOpenaircnRepo + ' \n' cmd = cmd + 'cd ' + logdirOAI5GRepo + '\n' cmd = cmd + 'git checkout ' + GitOAI5GRepoBranch + '\n' if GitOAI5GHeadVersion : cmd = cmd + 'git checkout ' + GitOAI5GHeadVersion + '\n' cmd = cmd + 'git_head=`git ls-remote |grep \'' + GitOAI5GRepoBranch + '\'` \n' cmd = cmd + 'git_head=($git_head) \n' cmd = cmd + 'git_head=${git_head[0]} \n' cmd = cmd + 'echo \"GitOAI5GHeadVersion_remote = $git_head\" \n' cmd = cmd + 'echo \"GitOAI5GHeadVersion_local = ' + GitOAI5GHeadVersion + '\" \n' if flag_skip_git_head_check==True: cmd = cmd + 'echo \"skipping GitHead check...\" \n ' else: cmd = cmd + 'if [ \"$git_head\" != \"'+ GitOAI5GHeadVersion + '\" ]; then echo \"error: Git openairinterface5g head version does not match\" ; fi \n' cmd = cmd + 'source oaienv' + '\n' if flag_skip_oai_install == False: cmd = cmd + 'source $OPENAIR_DIR/cmake_targets/tools/build_helper \n' cmd = cmd + 'echo \"Installing core OAI dependencies...Start\" \n' cmd = cmd + '$OPENAIR_DIR/cmake_targets/build_oai -I --install-optional-packages \n' cmd = cmd + 'echo \"Installing core OAI dependencies...Finished\" \n' #cmd = cmd + 'echo \"Installing BLADERF OAI dependencies...Start\" \n' #cmd = cmd + 'check_install_bladerf_driver \n' #cmd = cmd + 'echo \"Installing BLADERF OAI dependencies...Finished\" \n' #cmd = cmd + 'echo \"Installing USRP OAI dependencies...Start\" \n' #cmd = cmd + 'check_install_usrp_uhd_driver \n' #cmd = cmd + 'echo \"Installing USRP OAI dependencies...Finished\" \n' cmd = cmd + 'cd ' + logdirOpenaircnRepo + '\n' cmd = cmd + 'git checkout ' + GitOpenaircnRepoBranch + '\n' cmd = cmd + 'env |grep OPENAIR' + '\n' cmd = cmd + ' cd ' + logdir + '\n' cmd = cmd + 'mkdir -p ' + patchdir + '\n' cmd = cmd + ' ) > ' + setuplogfile + ' 2>&1 \n' #cmd = cmd + 'echo \' ' + cmd + '\' > ' + setup_script + ' 2>&1 \n ' #result = oai_list[index].send_recv(cmd, False, 300 ) write_file(setup_script, cmd, mode="w") tempThread = oaiThread(index, 'thread_setup_'+str(index)+'_' + MachineList[index] , MachineList[index] , user, pw, cmd, False, 3000) threads_init_setup.append(tempThread ) tempThread.start() index = index + 1 except Exception, e: print 'There is error in one of the commands to setup the machine '+ MachineList[index] error='' error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e) error = error + traceback.format_exc() print error sys.exit(1) #Now we wait for all the threads to complete index = 0 for t in threads_init_setup: t.join() port = 22 paramList=[] sftp_log = os.path.expandvars(locallogdir + '/sftp_module.log') #Now we copy patch files and apply them print( Fore.WHITE + " - Installating patch files on machine " + MachineList[index]) for patchFile in OAI5GpatchFileList: localfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/patches/')+patchFile.get('name') remotefile = logdirOAI5GRepo + '/cmake_targets/autotests/patches/'+patchFile.get('name') if patchFile.get('machine') == MachineList[index] or patchFile.get('machine') == None: if os.path.isfile(localfile): print( Fore.WHITE + "\t> PATCH FILE :"+localfile) paramList=[] paramList.append ( {"operation":'put', "localfile":localfile, "remotefile":remotefile} ) sftp_module (user, pw, MachineList[index], port, paramList, sftp_log) cmd = ' cd ' + logdirOAI5GRepo + ' ;git apply cmake_targets/autotests/patches/'+patchFile.get('name') res = oai_list[index].send_recv(cmd) #print res paramList=[] setuplogfile = logdir + '/setup_log_' + MachineList[index] + '_.txt' setup_script = locallogdir + '/setup_script_' + MachineList[index] + '_.txt' localfile = locallogdir + '/setup_log_' + MachineList[index] + '_.txt' remotefile = logdir + '/setup_log_' + MachineList[index] + '_.txt' sftp_log = os.path.expandvars(locallogdir + '/sftp_module.log') paramList.append ( {"operation":'get', "localfile":localfile, "remotefile":remotefile} ) #sftp_module (user, pw, MachineList[index], port, localfile, remotefile, sftp_log, "get") #Now we copy test_case_list.xml on the remote machines localfile = os.path.expandvars('$OPENAIR_DIR/cmake_targets/autotests/test_case_list.xml') remotefile = logdirOAI5GRepo + '/cmake_targets/autotests/test_case_list.xml' # paramList.append ( {"operation":'put', "localfile":localfile, "remotefile":remotefile} ) # sftp_module (user, pw, MachineList[index], port, paramList, sftp_log) sftp_module (user, pw, MachineList[index], port, paramList, sftp_log) cmd = ' cd ' + logdirOAI5GRepo + ' ; source oaienv ; env|grep OPENAIR \n' res = oai_list[index].send_recv(cmd) index = index +1 if os.path.exists(localfile) == 0: print "Setup log file <" + localfile + "> missing for machine <" + MachineList[index] + ">. Please check the setup log files. Exiting now" sys.exit(1) #Now we process all the test cases #Now we check if there was error in setup files status, out = commands.getstatusoutput('grep ' + ' -il \'error\' ' + locallogdir + '/setup_log*') if (out != '') : print "There is error in setup of machines" print "status = " + str(status) + "\n Check files for error = " + out print "Exiting now..." sys.exit(1) cleanOldProgramsAllMachines(oai_list, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop, '$HOME' , logdirOAI5GRepo) if cleanUpRemoteMachines == True: print "Error while cleaning Remote machines" print "Exiting now..." sys.exit(0) else: print (Fore.RED + " Skipping Machine preparation...") # ************************************************************************** # ************************************************************************** # STEP 2 - TEST SELECTION AND PARAMETRIZATION # ************************************************************************** # ************************************************************************** print (Fore.YELLOW + '\nStep 2 - TEST SELECTION') print (Fore.YELLOW + '-------------------------------------------------') print (Fore.WHITE + " Parsing test case list for test selection...") print (Fore.WHITE + " - TestCaseExclusionList : "+TestCaseExclusionList) print (Fore.WHITE + " - testcasegroup : "+testcasegroup) testcaseList=xmlRoot.findall('testCase') nb_total_testcases = 0 nb_total_testcases_softmodem = 0 nb_total_testcases_softmodem_noS1 = 0 nb_total_testcases_compilation = 0 nb_total_testcases_execution = 0 nb_run_testcases = 0 nb_run_testcases_softmodem = 0 nb_run_testcases_softmodem_noS1 = 0 nb_run_testcases_compilation = 0 nb_run_testcases_execution = 0 for testcase in testcaseList: try: testcasename = testcase.get('id') testcaseclass = testcase.findtext('class',default='') desc = testcase.findtext('desc',default='') nb_total_testcases+=1 run_flag = search_test_case_group(testcasename, testcasegroup, TestCaseExclusionList) if (run_flag): nb_run_testcases+=1 if testcaseclass == 'lte-softmodem' : nb_total_testcases_softmodem+=1 if (run_flag): nb_run_testcases_softmodem+=1 elif testcaseclass == 'lte-softmodem-noS1': nb_total_testcases_softmodem_noS1+=1 if (run_flag): nb_run_testcases_softmodem_noS1+=1 elif testcaseclass == 'compilation': nb_total_testcases_compilation+=1 if (run_flag): nb_run_testcases_compilation+=1 elif testcaseclass == 'execution': nb_total_testcases_execution+=1 if (run_flag): nb_run_testcases_execution+=1 else : print "Unknown test case class: " + testcaseclass sys.exit() except Exception, e: error='' error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e) error = error + '\n testcasename = ' + testcasename + '\n testcaseclass = ' + testcaseclass + '\n desc = ' + 'desc' + '\n' error = error + traceback.format_exc() print error print "Continuing to next test case..." print (Fore.WHITE + " Parsing done...") print (Fore.WHITE + " - Total number of test cases : "+str(nb_total_testcases)) print (Fore.WHITE + " - Total number class softmodem : "+str(nb_total_testcases_softmodem)) print (Fore.WHITE + " - Total number class softmodem-noS1 : "+str(nb_total_testcases_softmodem_noS1)) print (Fore.WHITE + " - Total number class compilation : "+str(nb_total_testcases_compilation)) print (Fore.WHITE + " - Total number class execution : "+str(nb_total_testcases_execution)) print print (Fore.WHITE + " - Selected number of test cases : "+str(nb_run_testcases)) print (Fore.WHITE + " - Selected class softmodem : "+str(nb_run_testcases_softmodem)) print (Fore.WHITE + " - Selected class softmodem_noS1 : "+str(nb_run_testcases_softmodem_noS1)) print (Fore.WHITE + " - Selected class compilation : "+str(nb_run_testcases_compilation)) print (Fore.WHITE + " - Selected class execution : "+str(nb_run_testcases_execution)) # ************************************************************************** # ************************************************************************** # STEP 3 - TEST CAMPAIGNS # ************************************************************************** # ************************************************************************** test_results = [] print (Fore.YELLOW + '\nStep 3 - TEST CAMPAIGNS') print (Fore.YELLOW + '-------------------------------------------------') print (Fore.WHITE + " Start test campaign...") threadListGlobal=[] run_count = 0 for testcase in testcaseList: try: testcasename = testcase.get('id') testcaseclass = testcase.findtext('class',default='') desc = testcase.findtext('desc',default='') #print "Machine list top level = " + ','.join(MachineList) if search_test_case_group(testcasename, testcasegroup, TestCaseExclusionList) == True: run_count+=1 print (Fore.WHITE + " ("+str(run_count).zfill(3)+"/"+str(nb_run_testcases).zfill(3)+") - test case "+testcasename+" : "), if testcaseclass == 'lte-softmodem' : eNBMachine = testcase.findtext('eNB',default='') UEMachine = testcase.findtext('UE',default='') EPCMachine = testcase.findtext('EPC',default='') #index_eNBMachine = MachineList.index(eNBMachine) #index_UEMachine = MachineList.index(UEMachine) #index_EPCMachine = MachineList.index(EPCMachine) if (eNBMachine not in MachineList)|(UEMachine not in MachineList)|(UEMachine not in MachineList): print "One of the machines is not in the machine list" print "eNBMachine : " + eNBMachine + "UEMachine : " + UEMachine + "EPCMachine : " + EPCMachine + "MachineList : " + ','.join(MachineList) print "testcasename = " + testcasename + " class = " + testcaseclass threadListGlobal = wait_testcaseclass_generic_threads(threadListGlobal, Timeout_execution) #cleanOldProgramsAllMachines(oai_list, CleanUpOldProgs, CleanUpAluLteBox, ExmimoRfStop) handle_testcaseclass_softmodem (testcase, CleanUpOldProgs, logdirOAI5GRepo, logdirOpenaircnRepo, MachineList, user, pw, CleanUpAluLteBox, ExmimoRfStop, nruns_lte_softmodem, Timeout_cmd ) #The lines below are copied from below to trace the failure of some of the machines in test setup. These lines below need to be removed in long term print "Creating xml file for overall results..." cmd = "cat $OPENAIR_DIR/cmake_targets/autotests/log/*/*.xml > $OPENAIR_DIR/cmake_targets/autotests/log/results_autotests.xml " res=os.system(cmd) os.system('sync') print "Now copying files to NFS Share" oai_localhost = openair('localdomain','localhost') oai_localhost.connect(user,pw) cmd = ' mkdir -p ' + NFSTestsResultsDir res = oai_localhost.send_recv(cmd) print "Copying files from GilabCI Runner Machine : " + host + " .locallogdir = " + locallogdir + ", NFSTestsResultsDir = " + NFSTestsResultsDir SSHSessionWrapper('localhost', user, None, pw , NFSTestsResultsDir , locallogdir, "put_all") oai_localhost.disconnect() elif testcaseclass == 'lte-softmodem-noS1' : eNBMachine = testcase.findtext('eNB',default='') UEMachine = testcase.findtext('UE',default='') if (eNBMachine not in MachineList)|(UEMachine not in MachineList): print "One of the machines is not in the machine list" print "eNBMachine : " + eNBMachine + "UEMachine : " + UEMachine + "MachineList : " + ','.join(MachineList) print "testcasename = " + testcasename + " class = " + testcaseclass threadListGlobal = wait_testcaseclass_generic_threads(threadListGlobal, Timeout_execution) handle_testcaseclass_softmodem_noS1 (testcase, CleanUpOldProgs, logdirOAI5GRepo, logdirOpenaircnRepo, MachineList, user, pw, CleanUpAluLteBox, ExmimoRfStop, nruns_lte_softmodem, Timeout_cmd ) #The lines below are copied from below to trace the failure of some of the machines in test setup. These lines below need to be removed in long term print "Creating xml file for overall results..." cmd = "cat $OPENAIR_DIR/cmake_targets/autotests/log/*/*.xml > $OPENAIR_DIR/cmake_targets/autotests/log/results_autotests.xml " res=os.system(cmd) os.system('sync') elif (testcaseclass == 'compilation'): threadListGlobal = handle_testcaseclass_generic (testcasename, threadListGlobal, CleanUpOldProgs, logdirOAI5GRepo, MachineListGeneric, user, pw, CleanUpAluLteBox,Timeout_execution, ExmimoRfStop) elif (testcaseclass == 'execution'): threadListGlobal = handle_testcaseclass_generic (testcasename, threadListGlobal, CleanUpOldProgs, logdirOAI5GRepo, MachineListGeneric, user, pw, CleanUpAluLteBox, Timeout_execution, ExmimoRfStop) else : print "Unknown test case class: " + testcaseclass sys.exit() except Exception, e: error='' error = error + ' In function: ' + sys._getframe().f_code.co_name + ': *** Caught exception: ' + str(e.__class__) + " : " + str( e) error = error + '\n testcasename = ' + testcasename + '\n testcaseclass = ' + testcaseclass + '\n desc = ' + 'desc' + '\n' error = error + traceback.format_exc() print error print "Continuing to next test case..." #sys.exit(1) print "Exiting the test cases execution now. Waiting for existing threads to complete..." for param in threadListGlobal: thread_id = param["thread_id"] thread_id.join() # ************************************************************************** # ************************************************************************** # STEP 4 - TEST REPORT PRODUCTION # ************************************************************************** # ************************************************************************** test_session_stop_time = datetime.datetime.now() print (Fore.YELLOW + '\nStep 4 - TEST REPORT PRODUCTION') print (Fore.YELLOW + '-------------------------------------------------') # XML test report # -------------------------------------------------------------------- print "Creating xml file for overall results..." cmd = "cat $OPENAIR_DIR/cmake_targets/autotests/log/*/*.xml > $OPENAIR_DIR/cmake_targets/autotests/log/results_autotests.xml " res=os.system(cmd) print "Now copying files to NFS Share" oai_localhost = openair('localdomain','localhost') oai_localhost.connect(user,pw) cmd = 'mkdir -p ' + NFSTestsResultsDir res = oai_localhost.send_recv(cmd) print "Copying files from GilabCI Runner Machine : " + host + " .locallogdir = " + locallogdir + ", NFSTestsResultsDir = " + NFSTestsResultsDir SSHSessionWrapper('localhost', user, None, pw , NFSTestsResultsDir , locallogdir, "put_all") cmd = "cat " + NFSTestsResultsDir + "/log/*/*.xml > " + NFSTestsResultsDir + "/log/results_autotests.xml" res = oai_localhost.send_recv(cmd) oai_localhost.disconnect() # HTML test report # -------------------------------------------------------------------- if flag_generate_html_report: print "Creating html test report..." report_dir = reportdir + '/'+ test_session_start_time.strftime("%Y-%m-%d_%H-%M")+"_"+host cmd = 'mkdir -p ' + report_dir result = os.system(cmd) cmd = 'cp -r '+locallogdir+' '+report_dir result = os.system(cmd) context = { 'report_path' : report_dir, 'test_session_start_time' : test_session_start_time, 'test_session_stop_time' : test_session_stop_time, 'test_session_duration' : test_session_stop_time-test_session_start_time, 'mtc_host' : host, 'user' : user, 'password' : pw, 'test_results' : test_results, } for test_result in test_results: cmd = 'mkdir -p ' + report_dir + '/'+ test_result['testcase_name'] result = os.system(cmd) report_file = report_dir + '/'+ test_result['testcase_name'] + '/'+ test_result['testcase_name']+ '_report.html' analyser.create_test_report_detailed_html(test_result, report_file ) # print test_result analyser.create_report_html(context) sys.exit()