run_exec_autotests.bash 17.2 KB
Newer Older
1 2
#!/bin/bash

3 4
if [ -s $OPENAIR_DIR/cmake_targets/tools/build_helper ] ; then
   source $OPENAIR_DIR/cmake_targets/tools/build_helper
5 6 7 8 9
else
   echo "Error: no file in the file tree: is OPENAIR_DIR variable set?"
   exit 1
fi

10 11
trap handle_ctrl_c INT

12
source $OPENAIR_DIR/cmake_targets/tools/test_helper
guptar's avatar
guptar committed
13

14

15
#SUDO="sudo -E "
guptar's avatar
guptar committed
16
tdir=$OPENAIR_DIR/cmake_targets/autotests
17 18
rm -fr $tdir/bin 
mkdir -p $tdir/bin
19
results_file="$tdir/log/results_autotests.xml"
guptar's avatar
guptar committed
20 21 22 23 24 25 26 27

updated=$(svn st -q $OPENAIR_DIR)
if [ "$updated" != "" ] ; then
	echo "some files are not in svn:\n $updated"
fi

cd $tdir 

28 29 30 31 32 33 34 35 36 37 38 39
#\param $1 -> name of test case
#\param $2 -> name of compilation program
#\param $3 -> arguments for compilation program
#\param $4 -> name of pre-executable to install kernel modules, etc
#\param $5 -> arguments of pre-executable
#\param $6 -> name of executable
#\param $7 -> arguments for running the program
#\param $8 -> search expression
#\param $9 -> search expression which should NOT be found (for ex. segmentation fault) 
#\param $10 -> number of runs
#\param $11 -> pre_compile program execution
#\param $12 -> class of the test case (compilation, execution)
40
#\param $13 -> output of compilation program that needs to be found for test case to pass
41
#\param $14 -> tags to help identify the test case for readability in output xml file
42
function test_compile() {
43 44 45

    xUnit_start
    test_case_name=$1
46 47
    log_dir=$tdir/log/$test_case_name
    log_file=$log_dir/test.$1.log.txt
48 49 50 51 52 53 54 55 56 57
    compile_prog=$2
    compile_args=$3
    pre_exec_file=$4
    pre_exec_args=$5
    exec_args=$7
    search_expr_array=("${!8}")
    search_expr_negative=$9
    nruns=${10}
    pre_compile_prog=${11}
    class=${12}
58
    compile_prog_out=${13}
59
    tags=${14}
60 61 62
    xmlfile_testcase=$log_dir/test.$1.xml
    #build_dir=$tdir/$1/build
    #exec_file=$build_dir/$6
63
    
64 65 66 67
    #compile_prog_out=`eval "echo $compile_prog_out"`
    #echo "compile_prog_out = $compile_prog_out"
    read -a compile_prog_out_array <<< "$compile_prog_out"

68 69
    #Temporary log file where execution log is stored.
    temp_exec_log=$log_dir/temp_log.txt
70 71
    rm -fr $log_dir
    mkdir -p $log_dir
72 73 74 75 76 77 78 79 80 81 82 83 84 85



    #echo "log_dir = $log_dir"
    #echo "log_file = $log_file"
    #echo "exec_file = $exec_file"
    #echo "exec_args = $exec_args"
    #echo "search_expr = $search_expr"
    #echo "pre_exec_file = $pre_exec_file"
    #echo "nruns = $nruns"
    echo "class = $class"
    
    compile_prog_array=()
    read -a compile_prog_array <<<"$compile_prog"
86
    
87 88
    #tags_array=()
    #read -a tags_array <<<"$tags"
89
    
90 91 92 93
    #pre_compile_prog_array=()
    #readarray -t pre_compile_prog_array <<< "$pre_compile_prog"
    result=1
    result_string=""
94 95
    for (( run_index=1; run_index <= $nruns; run_index++ ))
    do
96 97 98 99
       
    #tags_array_index=0
    #for pre_compile_prog_array_index in "${pre_compile_prog_array[@]}"  
    #do
100
    
101 102 103 104 105 106 107 108
    #for compile_prog_array_index in "${compile_prog_array[@]}"  
    #do
       echo "Compiling test case $test_case_name.${tags} Log file = $log_file"  
       date=`date`
       echo "<COMPILATION LOG file=$test_case_name.${tags} , Run = $run_index>, Date = $date " >> $log_file
       #rm -fr $build_dir
       #mkdir -p $build_dir
       cd $log_dir
109
       {   
110 111 112 113 114 115 116
          compile_log_dir=`eval echo \"$OPENAIR_DIR/cmake_targets/log/\"`
          echo "Removing compilation log files in $compile_log_dir"
          rm -frv $compile_log_dir
          echo "Executing $compile_prog $compile_prog_args ...."
          eval $compile_prog  $compile_prog_args
          echo "Copying compilation log files to test case log directory: $log_dir"
          cp -fvr $OPENAIR_DIR/cmake_targets/log/ $log_dir/compile_log
117 118 119
       }>> $log_file 2>&1
       echo "</COMPILATION LOG>" >> $log_file 2>&1
       if [ "$class" == "compilation" ]; then
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
         for compile_prog_out_index in ${compile_prog_out_array[@]}
            do 
                if  [ -s "$compile_prog_out_index" ]; then
                    let "result = result&1"
                    
                    echo_success "$test_case_name.${tags} RUN = $run_index $compile_prog_out_index = compiled"
                    
                else
                    let "result = result&0"
                            
                    echo_error "$test_case_name.${tags} RUN = $run_index $compile_prog_out_index failed"
                    
                 fi
            done #end of for loop compile_prog_out_index
         if [ "$result" == "1" ]; then
            result_string=$result_string" Run_$run_index = PASS"
         else   
            result_string=$result_string" Run_$run_index = FAIL"           
138
         fi
139

140
       fi
141 142 143
       #let "tags_array_index++"
    #done # End of for loop compile_prog_array
    #done # End of for loop (pre_compile_prog_array_index)
144
    done #End of for loop (run_index)
145 146 147 148 149 150 151 152 153 154 155 156 157
    
   
    #If for for some reason upper for loop does not execute, we fail the test case completely
    if [ "$result_string" == "" ]; then
      result=0
    fi
    if [ "$result" == "1" ]; then
      echo_success "$test_case_name.${tags} PASSED"
      xUnit_success "compilation" "$test_case_name.$tags" "PASS" "$result_string" "$xmlfile_testcase"
    else             
      echo_success "$test_case_name.${tags} FAILED"
      xUnit_success "compilation" "$test_case_name.$tags" "FAIL" "$result_string" "$xmlfile_testcase"
    fi
158 159 160
}


guptar's avatar
guptar committed
161 162 163 164 165 166 167 168 169

#\param $1 -> name of test case
#\param $2 -> name of compilation program
#\param $3 -> arguments for compilation program
#\param $4 -> name of pre-executable to install kernel modules, etc
#\param $5 -> arguments of pre-executable
#\param $6 -> name of executable
#\param $7 -> arguments for running the program
#\param $8 -> search expression
170 171
#\param $9 -> search expression which should NOT be found (for ex. segmentation fault) 
#\param $10 -> number of runs
172 173
#\param $11 -> pre_compile program execution
#\param $12 -> class of the test case (compilation, execution)
174 175
#\param $13 -> output of compilation program that needs to be found for test case to pass
#\param $14 -> tags to help identify the test case for readability in output xml file
176
#\param $15 => password for the user to run certain commands as sudo
guptar's avatar
guptar committed
177

178
function test_compile_and_run() {
guptar's avatar
guptar committed
179
    xUnit_start
180
    test_case_name=$1
181 182
    log_dir=$tdir/log/$test_case_name
    log_file=$log_dir/test.$1.log.txt
guptar's avatar
guptar committed
183 184 185 186
    compile_prog=$2
    compile_args=$3
    pre_exec_file=$4
    pre_exec_args=$5
187
    main_exec=$6
guptar's avatar
guptar committed
188
    exec_args=$7
guptar's avatar
guptar committed
189
    search_expr_array=("${!8}")
190 191
    search_expr_negative=$9
    nruns=${10}
192 193
    pre_compile_prog=${11}
    class=${12}
194 195
    compile_prog_out=${13}
    tags=${14}
196
    mypassword=${15}
guptar's avatar
guptar committed
197
    build_dir=$tdir/$1/build
198 199
    #exec_file=$build_dir/$6
    xmlfile_testcase=$log_dir/test.$1.xml
guptar's avatar
guptar committed
200 201 202
    #Temporary log file where execution log is stored.
    temp_exec_log=$log_dir/temp_log.txt
    
203 204 205 206 207 208 209
    rm -fr $log_dir
    mkdir -p $log_dir
    
    rm -fr $OPENAIR_DIR/cmake_targets/log

    echo "" > $temp_exec_log
    echo "" > $log_file
210 211 212
    #echo "log_dir = $log_dir"
    #echo "log_file = $log_file"
    #echo "exec_file = $exec_file"
213
    #echo "exec_args = $exec_args"
214 215 216
    #echo "search_expr = $search_expr"
    #echo "pre_exec_file = $pre_exec_file"
    #echo "nruns = $nruns"
217
    echo "class = $class"
guptar's avatar
guptar committed
218
    
219 220
    #compile_prog_array=()
    #read -a compile_prog_array <<<"$compile_prog"
221
  
222 223 224 225 226
    tags_array=()
    read -a tags_array <<<"$tags"
    
    main_exec_args_array=()
    readarray -t main_exec_args_array <<< "$exec_args"
227 228 229 230 231 232
    
    
    #for search_expr in "${compile_prog_array[@]}"  
    #do
       echo "Compiling test case $test_case_name Log file = $log_file"  
       echo "<COMPILATION LOG file=$log_file>" >> $log_file
233

234 235
       #rm -fr $build_dir
       #mkdir -p $build_dir
236

237
       cd $log_dir
238
       {   
239 240
          #eval $pre_compile_prog
          #cmake ..
241
          #rm -fv $exec_file
242 243 244 245
          echo "Executing $compile_prog $compile_args" >> $log_file
          eval "$compile_prog $compile_args"
          echo "Copying compilation log files to test case log directory: $log_dir"
          cp -fvr $OPENAIR_DIR/cmake_targets/log/ $log_dir/compile_log
246 247
       }>> $log_file 2>&1
       echo "</COMPILATION LOG>" >> $log_file 2>&1
248
    #done
249

250 251
    #process the test case if it is that of execution
    if [ "$class" == "execution" ]; then
252 253 254
      tags_array_index=0
      for main_exec_args_array_index in "${main_exec_args_array[@]}"  
      do
255 256 257 258
        global_result=1
        result_string=""
        
       for (( run_index=1; run_index <= $nruns; run_index++ ))
259
        do
260 261
          temp_exec_log=$log_dir/test.$test_case_name.${tags_array[$tags_array_index]}.run_$run_index
          echo "" > $temp_exec_log
guptar's avatar
guptar committed
262

263 264 265 266 267
          echo "Executing test case $test_case_name.${tags_array[$tags_array_index]}, Run Index = $run_index, Execution Log file = $temp_exec_log"

          echo "-----------------------------------------------------------------------------" >> $temp_exec_log  2>&1
          echo "<EXECUTION LOG Test Case = $test_case_name.${tags_array[$tags_array_index]}, Run = $run_index >" >> $temp_exec_log  2>&1
           
268
          if [ -n "$pre_exec_file" ]; then
269 270
            {  echo " Executing $pre_exec_file $pre_exec_args " 
               eval " echo '$mypassword' |sudo -S -E $pre_exec_file $pre_exec_args " ; }>> $temp_exec_log  2>&1
271

272
          fi
273 274 275
          echo "Executing $main_exec $main_exec_args_array_index "
          echo "Executing $main_exec $main_exec_args_array_index " >> $temp_exec_log
          { eval "$main_exec $main_exec_args_array_index" ;} >> $temp_exec_log  2>&1
276

277
          echo "</EXECUTION LOG Test Case = $test_case_name.${tags_array[$tags_array_index]},  Run = $run_index >" >> $temp_exec_log  2>&1
278
          cat $temp_exec_log >> $log_file  2>&1
279
          
guptar's avatar
guptar committed
280
    
281 282 283
          result=1
          for search_expr in "${search_expr_array[@]}"
          do
guptar's avatar
guptar committed
284
     
285
            search_result=`grep -E "$search_expr" $temp_exec_log`
guptar's avatar
guptar committed
286

287 288
            #echo "search_expr  =   $search_expr"
            #echo "search_result = $search_result"
guptar's avatar
guptar committed
289

290 291 292 293 294 295
            if [ -z "$search_result" ]; then
              let "result = result & 0"
            else
              let "result = result & 1"
            fi
          done
296 297 298 299 300 301 302
          
          #If we find a negative search result then there is crash of program and test case is failed even if above condition is true
          search_result=`grep -iE "$search_expr_negative" $temp_exec_log`
          if [ -n "$search_result" ]; then
            result=0
          fi
          let "global_result = global_result & result"
guptar's avatar
guptar committed
303

304
          #echo "result = $result"
305 306 307
          
          #this is a result of this run
          #test_case_result=""
308
          if [ "$result" -eq "0" ]; then
309 310
            result_string=$result_string" Run_$run_index =FAIL"
            echo_error "$test_case_name.${tags_array[$tags_array_index]} RUN = $run_index Result = FAIL"
311
          fi
guptar's avatar
guptar committed
312

313
          if [ "$result" -eq "1" ]; then
314 315
            result_string=$result_string" Run_$run_index =PASS"
            echo_success "$test_case_name.${tags_array[$tags_array_index]} RUN = $run_index Result = PASS"
316
          fi
guptar's avatar
guptar committed
317

318
          
319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
        done #End of for loop (nindex)

       echo " Result String = $result_string" 

       if [ "$result_string" == "" ]; then
           echo_error "execution $test_case_name.$compile_prog.${tags_array[$tags_array_index]} Run_Result = \"$result_string\"  Result = FAIL"
	   xUnit_success "execution" "$test_case_name.${tags_array[$tags_array_index]}" "FAIL" "$result_string" "$xmlfile_testcase"
       else
        if [ "$global_result" == "0" ]; then
           echo_error "execution $test_case_name.${tags_array[$tags_array_index]} Run_Result = \"$result_string\" Result =  FAIL"
           xUnit_fail "execution" "$test_case_name.${tags_array[$tags_array_index]}" "FAIL" "$result_string" "$xmlfile_testcase"
        fi

        if [ "$global_result" == "1" ]; then
            echo_success "execution $test_case_name.${tags_array[$tags_array_index]} Run_Result = \"$result_string\"  Result = PASS "
	    xUnit_success "execution" "$test_case_name.${tags_array[$tags_array_index]}" "PASS" "$result_string"   "$xmlfile_testcase"     
        fi  
       fi

338
       let "tags_array_index++"
339
     done 
340
   fi
341
   rm -fr $build_dir
guptar's avatar
guptar committed
342 343
}

344 345 346
dbin=$OPENAIR_DIR/cmake_targets/autotests/bin
dlog=$OPENAIR_DIR/cmake_targets/autotests/log

guptar's avatar
guptar committed
347

348
function print_help() {
349 350 351 352 353 354 355 356 357 358
 echo_info '
This program runs automated test case system for OpenAirInterface
You should have ubuntu 14.xx, updated, and the Linux kernel >= 3.14
Options
-h | --help
   This help
-g | --run-group
   Run test cases in a group. For example, ./run_exec_autotests "0101* 010102"
'
}
359

360
function main () {
361
RUN_GROUP=0
362 363 364
test_case_group=""
test_case_group_array=()
test_case_array=()
365 366
echo_info "Note that the user should be sudoer for executing certain commands, for example loading kernel modules"
read -s -p "Enter Password: " mypassword
367 368 369 370 371 372

until [ -z "$1" ]
  do
    case "$1" in
       -g | --run-group)
            RUN_GROUP=1
373 374
            test_case_group=$2
            echo_info "Will execute test cases only in group $test_case_group"
375 376 377 378 379 380 381 382 383 384
            shift 2;;
        -h | --help)
            print_help
            exit 1;;
	*)
	    print_help
            echo_fatal "Unknown option $1"
            break;;
   esac
  done
385

386

387 388
xml_conf="$OPENAIR_DIR/cmake_targets/autotests/test_case_list.xml"

389
test_case_list=`xmlstarlet sel -T -t -m /testCaseList/testCase -s A:N:- "@id" -v "@id" -n $xml_conf`
390 391 392 393 394

echo "test_case_list = $test_case_list"

readarray -t test_case_array <<<"$test_case_list"

395 396
read -a test_case_group_array <<< "$test_case_group"
 
397 398
for search_expr in "${test_case_array[@]}"
  do
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
    flag_run_test_case=0
    # search if this test case needs to be executed
    if [ "$RUN_GROUP" -eq "1" ]; then
       for search_group in "${test_case_group_array[@]}"
       do  
          if [[ $search_expr == $search_group ]];then
             flag_run_test_case=1
             echo_info "Test case $search_expr match found in group"
             break
          fi
       done
    else
       flag_run_test_case=1
    fi
       
    #We skip this test case if it is not in the group list
    if [ "$flag_run_test_case" -ne "1" ]; then
      continue
    fi

419
    name=$search_expr
420 421 422 423 424 425 426 427 428 429 430 431 432 433
    class=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/class" $xml_conf`
    desc=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/desc" $xml_conf`
    pre_compile_prog=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/pre_compile_prog" $xml_conf`
    compile_prog=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/compile_prog" $xml_conf`
    compile_prog_args=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/compile_prog_args" $xml_conf`
    pre_exec=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/pre_exec" $xml_conf`
    pre_exec_args=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/pre_exec_args" $xml_conf`
    main_exec=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/main_exec" $xml_conf`
    main_exec_args=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/main_exec_args" $xml_conf`
    search_expr_true=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/search_expr_true" $xml_conf`
    search_expr_false=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/search_expr_false" $xml_conf`
    nruns=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/nruns" $xml_conf`
    compile_prog_out=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/compile_prog_out" $xml_conf`
    tags=`xmlstarlet sel -t -v "/testCaseList/testCase[@id='$search_expr']/tags" $xml_conf`
434

435 436 437 438 439
    echo "class = $class"
    echo "name = $name"
    echo "Description = $desc"
    echo "pre_compile_prog = $pre_compile_prog"
    echo "compile_prog = $compile_prog"
440 441
    echo "compile_prog_args = $compile_prog_args"
    echo "compile_prog_out = $compile_prog_out"
442 443 444 445 446 447 448 449 450
    echo "pre_exec = $pre_exec"
    echo "pre_exec_args = $pre_exec_args"
    echo "main_exec = $main_exec"
    echo "main_exec_args = $main_exec_args"
    echo "search_expr_true = $search_expr_true"
    echo "search_expr_false = $search_expr_false"
    echo "nruns = $nruns"

    #eval $pre_exec
451
    compile_prog_out=`eval echo \"$compile_prog_out\"`
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469

    search_array_true=()

    IFS=\"                  #set the shell's field separator
    set -f                  #don't try to glob 
    #set -- $search_expr_true             #split on $IFS
    for i in $search_expr_true
      do echo "i = $i"
        if [ -n "$i" ] && [ "$i" != " " ]; then
          search_array_true+=("$i")
          #echo "inside i = \"$i\" "
        fi 
      done
    unset IFS 

    #echo "arg1 = ${search_array_true[0]}"
    #echo " arg2 = ${search_array_true[1]}"
    if [ "$class" == "compilation" ]; then
470 471
        test_compile "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class" "$compile_prog_out" "$tags"
    elif  [ "$class" == "execution" ]; then
472
        test_compile_and_run "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class" "$compile_prog_out" "$tags" "$mypassword" 
473
    else
474
        echo "Unexpected class of test case...Skipping the test case $name ...."
475 476 477 478 479 480 481 482
    fi

    done
    
    

}

483
main "$@"
484 485 486 487 488 489 490 491 492

xUnit_write "$results_file"

echo "Test Results are written to $results_file"

exit