Merge remote-tracking branch 'origin/ci-collect-ue-logs' into integration_2025_w32 (!3560)

CI: Refactor log collection, collect also (COTS) UE logs

Refactor log handling in CI: collect according to currently used XML files. This
makes it homogeneous in the pipelines, and avoids accidental overwriting (as
happens in phytest-timing), as well as harmonizing the log placement
(e.g., no YAML lookup in Ping/Iperf).

Allow to trace and archive UE logs (e.g., Quectel via QLog, Amarisoft UE logs).
By default, no logs will be collected as of this MR, as many files are big.

Various cleanup in Jenkinsfiles, some bugfixes, add unit tests.

ci-scripts/Jenkinsfile-tmp-full-ran

@@ -216,24 +216,6 @@ pipeline {
         }
         stage('Log Collection') {
             parallel {
-                stage('Log Collection (eNB - Build)') {
-                    steps {
-                        withCredentials([
-                             [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
-                        ]) {
-                            echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
-                            sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
-
-                            echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
-                            sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
-                        }
-                        script {
-                            if(fileExists("build.log.${env.BUILD_ID}.zip")) {
-                                archiveArtifacts "build.log.${env.BUILD_ID}.zip"
-                            }
-                        }
-                    }
-                }
                 stage('Log Collection (eNB - Run)') {
                     steps {
                         withCredentials([

ci-scripts/Jenkinsfile-tmp-multi-enb

@@ -222,24 +222,6 @@ pipeline {
         }
         stage('Log Collection') {
             parallel {
-                stage('Log Collection (eNB - Build)') {
-                    steps {
-                        withCredentials([
-                             [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
-                        ]) {
-                            echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
-                            sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
-
-                            echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
-                            sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
-                        }
-                        script {
-                            if(fileExists("build.log.${env.BUILD_ID}.zip")) {
-                                archiveArtifacts "build.log.${env.BUILD_ID}.zip"
-                            }
-                        }
-                    }
-                }
                 stage('Log Collection (eNB - Run)') {
                     steps {
                         withCredentials([

ci-scripts/Jenkinsfile-tmp-multi-enb-nsa

@@ -210,24 +210,6 @@ pipeline {
     }
     stage('Log Collection') {
       parallel {
-        stage('Log Collection (eNB - Build)') {
-          steps {
-            withCredentials([
-               [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
-            ]) {
-              echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
-              sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
-
-              echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
-              sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
-            }
-            script {
-              if(fileExists("build.log.${env.BUILD_ID}.zip")) {
-                archiveArtifacts "build.log.${env.BUILD_ID}.zip"
-              }
-            }
-          }
-        }
         stage('Log Collection (eNB - Run)') {
           steps {
             withCredentials([

ci-scripts/Jenkinsfile-tmp-ran

@@ -227,24 +227,6 @@ pipeline {
         }
         stage('Log Collection') {
             parallel {
-                stage('Log Collection (eNB - Build)') {
-                    steps {
-                        withCredentials([
-                             [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
-                        ]) {
-                            echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
-                            sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
-
-                            echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
-                            sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
-                        }
-                        script {
-                            if(fileExists("build.log.${env.BUILD_ID}.zip")) {
-                                archiveArtifacts "build.log.${env.BUILD_ID}.zip"
-                            }
-                        }
-                    }
-                }
                 stage('Log Collection (eNB - Run)') {
                     steps {
                         withCredentials([

ci-scripts/ci_infra.yaml

@@ -5,10 +5,12 @@ idefix:
   AttachScript: sudo python3 ci_ctl_qtel.py /dev/ttyUSB2 wup
   DetachScript: sudo python3 ci_ctl_qtel.py /dev/ttyUSB2 detach
   NetworkScript: ip a show dev wwan0
+  #Tracing:
+  #  Start: 'sudo rm -f ci_qlog/*; nohup sudo -b QLog/QLog -s ci_qlog -f NR5G.cfg &'
+  #  Stop: 'sudo killall QLog'
+  #  Collect: 'sudo mv ci_qlog/* %%log_dir%%/'
   IF: wwan0
   MTU: 1500
-  Trace: True
-  LogStore: /media/usb-drive/ci_qlogs
 
 up2:
   Host: up2
@@ -46,7 +48,6 @@ adb_ue_1:
   NetworkScript: adb -s R3CM40LZPHT shell "ip address show | grep rmnet_data0"
   CmdPrefix: adb -s R3CM40LZPHT shell
   MTU: 1500
-  LogStore: /media/usb-drive/ci_adb_1-logs
 adb_ue_2:
   Host: nano
   InitScript: /home/oaicicd/ci_ctl_adb.sh initialize 5200c00db4413517
@@ -59,7 +60,6 @@ adb_ue_2:
   NetworkScript: adb -s 5200c00db4413517 shell "ip address show | grep rmnet"
   CmdPrefix: adb -s 5200c00db4413517 shell
   MTU: 1500
-  LogStore: /media/usb-drive/ci_adb_2-logs
 
 oc-cn5g:
   Host: avra
@@ -150,6 +150,10 @@ amarisoft_ue:
   InitScript: /root/lteue-linux-2025-03-15/lteue /root/oaicicd/ran_sa_aw2s_asue_20MHz/aw2s-multi-00102-20.cfg &
   TermScript: /root/lteue-linux-2025-03-15/ws.js -t 10 127.0.0.1:9002 '{"message":"quit"}' || killall -KILL lteue-avx2
   NetworkScript: ip netns exec ue1 ip a show dev pdn0
+  #Tracing:
+  #  Start: '' # nothing to be done
+  #  Stop: '' # nothing to be done
+  #  Collect: 'cp /tmp/ue.log %%log_dir%%'
 amarisoft_ue_2x2:
   Host: amariue
   InitScript: /root/lteue-linux-2025-03-15/lteue /root/oaicicd/ran_sa_aw2s_asue_20MHz_2x2/aw2s-multi-00102-2x2-v2.cfg &

ci-scripts/cls_analysis.py

@@ -84,7 +84,7 @@ class Analysis():
 
 		return success,msg
 
-	def analyze_oc_physim(result_junit, details_json):
+	def analyze_oc_physim(result_junit, details_json, logPath):
 		try:
 			tree = ET.parse(result_junit)
 			root = tree.getroot()
@@ -124,7 +124,7 @@ class Analysis():
 			output = test.findtext("system-out")
 			output_check = "exceeds the threshold" not in output
 			# collect logs
-			log_dir = f'../cmake_targets/log/{test_exec}'
+			log_dir = f'{logPath}/{test_exec}'
 			os.makedirs(log_dir, exist_ok=True)
 			with open(f'{log_dir}/{test_name}.log', 'w') as f:
 				f.write(output)

ci-scripts/cls_ci_helper.py

+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+
+import os
+from typing import NamedTuple
+import logging
+
+class TestCaseCtx(NamedTuple):
+    count: int
+    test_id: int
+    logPath: str
+
+    def Default(logPath):
+        return TestCaseCtx(123, 112233, logPath)
+    def baseFilename(self):
+        # typically, the test ID is of form 001234 (6 digits)
+        return f"{self.logPath}/{self.count}-{self.test_id:06d}"
+
+def archiveArtifact(cmd, ctx, remote_path):
+    base = os.path.basename(remote_path)
+    local = f"{ctx.baseFilename()}-{base}"
+    logging.info(f"Archive artifact '{local}'")
+    success = cmd.copyin(remote_path, local)
+    if success:
+        cmd.run(f'rm {remote_path}', silent=True)
+    return local if success else None

ci-scripts/cls_corenetwork.py

@@ -29,6 +29,7 @@ import yaml
 import re
 
 import cls_cmd
+from cls_ci_helper import archiveArtifact
 
 def listify(s):
 	if s is None:
@@ -114,8 +115,7 @@ class CoreNetwork:
 		logging.info(f'deployed core network {self}, pingable IP address {ip}')
 		return True, output
 
-	def _collect_logs(self, log_dir):
-		logging.info(f'collecting logs into (local) {log_dir}')
+	def _collect_logs(self, ctx):
 		remote_dir = "/tmp/cn-undeploy-logs"
 		with cls_cmd.getConnection(self._host) as c:
 			# create a directory for log collection
@@ -134,18 +134,17 @@ class CoreNetwork:
 				logging.error("cannot enumerate log files")
 				return []
 			log_files = []
-			# copy them to the executor one by one, and store in log_dir
+			# copy them to the executor one by one
 			for f in ret.stdout.split("\n"):
-				l = f.replace(remote_dir, log_dir)
-				c.copyin(f, l)
-				log_files.append(l)
+				name = archiveArtifact(c, ctx, f)
+				log_files.append(name)
 			c.run(f'rm -rf {remote_dir}')
 			return log_files
 
-	def undeploy(self, log_dir=None):
+	def undeploy(self, ctx=None):
 		log_files = []
-		if log_dir is not None:
-			log_files = self._collect_logs(log_dir)
+		if ctx is not None:
+			log_files = self._collect_logs(ctx)
 		else:
 			logging.warning("no directory for log collection specified, cannot retrieve core network logs")
 		logging.info(f'undeploy core network {self}')

ci-scripts/cls_module.py

@@ -32,6 +32,7 @@ import re
 import yaml
 
 import cls_cmd
+from cls_ci_helper import archiveArtifact
 
 class Module_UE:
 
@@ -59,26 +60,31 @@ class Module_UE:
 			}
 			self.interface = m.get('IF')
 			self.MTU = m.get('MTU')
-			self.trace = m.get('trace') == True
-			self.logStore = m.get('LogStore')
 			self.cmd_prefix = m.get('CmdPrefix')
 			logging.info(f'initialized UE {self} from {filename}')
 
+			t = m.get('Tracing')
+			self.trace = t is not None
+			if self.trace:
+				if t.get('Start') is None or t.get('Stop') is None or t.get('Collect')is None :
+					raise ValueError("need to have Start/Stop/Collect for tracing")
+				self.cmd_dict["traceStart"] = t.get('Start')
+				self.cmd_dict["traceStop"] = t.get('Stop')
+				self._logCollect = t.get('Collect')
+				if "%%log_dir%%" not in self._logCollect:
+					raise ValueError(f"(At least one) LogCollect expression for {module_name} must contain \"%%log_dir%%\"")
+
 	def __str__(self):
-		return f"{self.module_name}@{self.host} [IP: {self.getIP()}]"
+		return f"{self.module_name}@{self.host}"
 
 	def __repr__(self):
 		return self.__str__()
 
-	def _command(self, cmd, silent = False):
+	def _command(self, cmd, silent=False, reportNonZero=True):
 		if cmd is None:
 			raise Exception("no command provided")
-		if self.host == "" or self.host == "localhost":
-			c = cls_cmd.LocalCmd()
-		else:
-			c = cls_cmd.RemoteCmd(self.host)
-		response = c.run(cmd, silent=silent)
-		c.close()
+		with cls_cmd.getConnection(self.host) as c:
+			response = c.run(cmd, silent=silent, reportNonZero=reportNonZero)
 		return response
 
 #-----------------$
@@ -86,25 +92,23 @@ class Module_UE:
 #-----------------$
 
 	def initialize(self):
-		if self.trace:
-			raise Exception("UE tracing not implemented yet")
-			self._enableTrace()
 		# we first terminate to make sure the UE has been stopped
 		if self.cmd_dict["detach"]:
 			self._command(self.cmd_dict["detach"], silent=True)
 		self._command(self.cmd_dict["terminate"], silent=True)
 		ret = self._command(self.cmd_dict["initialize"])
 		logging.info(f'For command: {ret.args} | return output: {ret.stdout} | Code: {ret.returncode}')
+		if self.trace:
+			self._enableTrace()
 		# Here each UE returns differently for the successful initialization, requires check based on UE
 		return ret.returncode == 0
 
 
-	def terminate(self):
+	def terminate(self, ctx=None):
 		self._command(self.cmd_dict["terminate"])
-		if self.trace:
-			raise Exception("UE tracing not implemented yet")
+		if self.trace and ctx is not None:
 			self._disableTrace()
-			return self._logCollect()
+			return self._collectTrace(ctx)
 		return None
 
 	def attach(self, attach_tries = 4, attach_timeout = 60):
@@ -113,10 +117,11 @@ class Module_UE:
 			self._command(self.cmd_dict["attach"])
 			timeout = attach_timeout
 			logging.debug("Waiting for IP address to be assigned")
+			ip = self.getIP(silent=False, reportNonZero=True)
 			while timeout > 0 and not ip:
-				time.sleep(5)
-				timeout -= 5
-				ip = self.getIP()
+				time.sleep(1)
+				timeout -= 1
+				ip = self.getIP(silent=True, reportNonZero=False)
 			if ip:
 				break
 			logging.warning(f"UE did not receive IP address after {attach_timeout} s, detaching")
@@ -160,8 +165,8 @@ class Module_UE:
 			logging.error(message)
 			return False
 
-	def getIP(self):
-		output = self._command(self.cmd_dict["getNetwork"], silent=True)
+	def getIP(self, silent=True, reportNonZero=True):
+		output = self._command(self.cmd_dict["getNetwork"], silent=silent, reportNonZero=reportNonZero)
 		result = re.search(r'inet (?P<ip>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)', output.stdout)
 		if result and result.group('ip'):
 			ip = result.group('ip')
@@ -191,10 +196,33 @@ class Module_UE:
 		return self.cmd_prefix if self.cmd_prefix else ""
 
 	def _enableTrace(self):
-		raise Exception("not implemented")
+		logging.info(f'UE {self}: start UE tracing')
+		self._command(self.cmd_dict["traceStart"])
 
 	def _disableTrace(self):
-		raise Exception("not implemented")
-
-	def _logCollect(self):
-		raise Exception("not implemented")
+		logging.info(f'UE {self}: stop UE tracing')
+		self._command(self.cmd_dict["traceStop"])
+
+	def _collectTrace(self, ctx):
+		remote_dir = "/tmp/ue-trace-logs"
+		with cls_cmd.getConnection(self.host) as c:
+			# create a directory for log collection
+			c.run(f'rm -rf {remote_dir}')
+			ret = c.run(f'mkdir {remote_dir}')
+			if ret.returncode != 0:
+				logging.error("cannot create directory for log collection")
+				return []
+			log_cmd = self._logCollect.replace('%%log_dir%%', remote_dir)
+			self._command(log_cmd)
+			# enumerate collected files
+			ret = c.run(f'ls {remote_dir}/*')
+			if ret.returncode != 0:
+				logging.error("cannot enumerate log files")
+				return []
+			log_files = []
+			# copy them to the executor one by one, and store in log_dir
+			for f in ret.stdout.split("\n"):
+				name = archiveArtifact(c, ctx, f)
+				log_files.append(name)
+			c.run(f'rm -rf {remote_dir}')
+			return log_files

ci-scripts/cls_native.py

@@ -28,27 +28,24 @@ import cls_cmd
 import cls_oai_html
 import cls_analysis
 import constants as CONST
+from cls_ci_helper import archiveArtifact
 
-LOG_PATH_PHYSIM = 'phy_sim_logs'
 DPDK_PATH = '/opt/dpdk-t2-22.11.0'
 
 class Native():
 
-	def Build(test_case, HTML, host, directory, options):
+	def Build(ctx, test_case, HTML, host, directory, options):
 		logging.debug(f'Building on server: {host}')
 		HTML.testCase_id = test_case
 
 		with cls_cmd.getConnection(host) as ssh:
 			base = f"{directory}/cmake_targets"
-			ret = ssh.run(f"C_INCLUDE_PATH={DPDK_PATH}/include/ PKG_CONFIG_PATH={DPDK_PATH}/lib64/pkgconfig/ {base}/build_oai {options} > {base}/log/build_oai.log", timeout=900)
+			ret = ssh.run(f"C_INCLUDE_PATH={DPDK_PATH}/include/ PKG_CONFIG_PATH={DPDK_PATH}/lib64/pkgconfig/ {base}/build_oai {options} > {base}/build_oai.log", timeout=900)
 			success = ret.returncode == 0
-			logs = ssh.run(f"cat {base}/log/build_oai.log", silent=True)
+			logs = ssh.run(f"cat {base}/build_oai.log", silent=True)
 			logging.debug(f"build finished with code {ret.returncode}, output:\n{logs.stdout}")
 
-			# create log directory, and copy build logs
-			target = f"{base}/build_log_{test_case}/"
-			ssh.run(f"mkdir -p {target}")
-			ssh.run(f"mv {base}/log/* {target}")
+			archiveArtifact(ssh, ctx, f'{base}/build_oai.log')
 
 			# check if build artifacts are there
 			# NOTE: build_oai should fail with exit code if it could not build, but it does not
@@ -68,15 +65,14 @@ class Native():
 			HTML.CreateHtmlTestRow(options, 'KO', CONST.ALL_PROCESSES_OK)
 		return success
 
-	def Run_Physim(HTML, host, directory, options, physim_test, threshold):
+	def Run_Physim(ctx, HTML, host, directory, options, physim_test, threshold):
 		logging.debug(f'Runnin {physim_test} on server: {host}')
 		workSpacePath = f'{directory}/cmake_targets'
-		os.system(f'mkdir -p ./{LOG_PATH_PHYSIM}')
-		runLogFile=f'physim_{HTML.testCase_id}.log'
+		runLogFile=f'{workSpacePath}/physim.log'
 		with cls_cmd.getConnection(host) as cmd:
-			cmd.run(f'sudo LD_LIBRARY_PATH=.:{DPDK_PATH}/lib64/ {workSpacePath}/ran_build/build/{physim_test} {options} >> {workSpacePath}/{runLogFile}')
-			cmd.copyin(src=f'{workSpacePath}/{runLogFile}', tgt=f'{LOG_PATH_PHYSIM}/{runLogFile}')
-		success, msg = cls_analysis.Analysis.analyze_physim(f'{LOG_PATH_PHYSIM}/{runLogFile}', physim_test, options, threshold)
+			cmd.run(f'sudo LD_LIBRARY_PATH=.:{DPDK_PATH}/lib64/ {workSpacePath}/ran_build/build/{physim_test} {options} >> {runLogFile}')
+			physim_file = archiveArtifact(cmd, ctx, runLogFile)
+		success, msg = cls_analysis.Analysis.analyze_physim(physim_file, physim_test, options, threshold)
 		if success:
 			HTML.CreateHtmlTestRowQueue(options, 'OK', [msg])
 		else:

ci-scripts/cls_static_code_analysis.py

@@ -43,6 +43,7 @@ from pathlib import Path
 import helpreadme as HELP
 import constants as CONST
 import cls_cmd
+from cls_ci_helper import archiveArtifact
 
 #-----------------------------------------------------------
 # Class Declaration
@@ -80,7 +81,7 @@ class StaticCodeAnalysis():
 		self.eNBPassword = ''
 		self.eNBSourceCodePath = ''
 
-	def CppCheckAnalysis(self, HTML):
+	def CppCheckAnalysis(self, ctx, HTML):
 		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
 			HELP.GenericHelp(CONST.Version)
 			sys.exit('Insufficient Parameter')
@@ -103,7 +104,7 @@ class StaticCodeAnalysis():
 			full_ran_repo_name = self.ranRepository + '.git'
 
 		cmd.cd(lSourcePath)
-		logDir = f'{lSourcePath}/cmake_targets/build_log_{self.testCase_id}'
+		logDir = f'{lSourcePath}/cmake_targets/log'
 		cmd.run(f'mkdir -p {logDir}')
 		cmd.run('docker image rm oai-cppcheck:bionic oai-cppcheck:focal')
 		cmd.run(f'sed -e "s@xenial@bionic@" {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.xenial > {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.bionic')
@@ -112,31 +113,21 @@ class StaticCodeAnalysis():
 		cmd.run(f'docker build --tag oai-cppcheck:focal --file {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.focal . > {logDir}/cppcheck-focal.txt 2>&1')
 		cmd.run('docker image rm oai-cppcheck:bionic oai-cppcheck:focal')
 
-		# Analyzing the logs
-		cmd.copyin(f'{logDir}/cppcheck-bionic.txt', 'cppcheck-bionic.txt')
-		cmd.copyin(f'{logDir}/cppcheck-focal.txt', 'cppcheck-focal.txt')
+		bionic = archiveArtifact(cmd, ctx, f'{logDir}/cppcheck-bionic.txt')
+		focal = archiveArtifact(cmd, ctx, f'{logDir}/cppcheck-focal.txt')
 		cmd.close()
 
 		CCR = CppCheckResults()
 		CCR_ref = CppCheckResults()
 		vId = 0
 		for variant in CCR.variants:
-			refAvailable = False
-			if self.ranAllowMerge:
-				refFolder = str(Path.home()) + '/cppcheck-references'
-				if (os.path.isfile(refFolder + '/cppcheck-'+ variant + '.txt')):
-					refAvailable = True
-					with open(refFolder + '/cppcheck-'+ variant + '.txt', 'r') as refFile:
-						for line in refFile:
-							ret = re.search(' (?P<nb_errors>[0-9\.]+) errors', str(line))
-							if ret is not None:
-								CCR_ref.nbErrors[vId] = int(ret.group('nb_errors'))
-							ret = re.search(' (?P<nb_warnings>[0-9\.]+) warnings', str(line))
-							if ret is not None:
-								CCR_ref.nbWarnings[vId] = int(ret.group('nb_warnings'))
-			if (os.path.isfile('./cppcheck-'+ variant + '.txt')):
+			filename = ctx.baseFilename() + '-cppcheck-'+ variant + '.txt'
+			logging.info(f"will check file '{filename}'")
+			if not os.path.isfile(filename):
+				raise FileNotFoundError(f"{filename} is not a file")
+			else:
 				xmlStart = False
-				with open('./cppcheck-'+ variant + '.txt', 'r') as logfile:
+				with open(filename, 'r') as logfile:
 					for line in logfile:
 						ret = re.search('cppcheck version="(?P<version>[0-9\.]+)"', str(line))
 						if ret is not None:
@@ -185,26 +176,6 @@ class StaticCodeAnalysis():
 			vMsg += '   Wrong Scanf Nb Args:             ' + str(CCR.nbWrongScanfArg[vId]) + '\n'
 			for vLine in vMsg.split('\n'):
 				logging.debug(vLine)
-			if self.ranAllowMerge and refAvailable:
-				if CCR_ref.nbErrors[vId] == CCR.nbErrors[vId]:
-					logging.debug('   No change in number of errors')
-				elif CCR_ref.nbErrors[vId] > CCR.nbErrors[vId]:
-					logging.debug('   Good! Decrease in number of errors')
-				else:
-					logging.debug('   Bad! increase in number of errors')
-				if CCR_ref.nbWarnings[vId] == CCR.nbWarnings[vId]:
-					logging.debug('   No change in number of warnings')
-				elif CCR_ref.nbWarnings[vId] > CCR.nbWarnings[vId]:
-					logging.debug('   Good! Decrease in number of warnings')
-				else:
-					logging.debug('   Bad! increase in number of warnings')
-			# Create new reference file
-			if not self.ranAllowMerge:
-				refFolder = str(Path.home()) + '/cppcheck-references'
-				if not os.path.isdir(refFolder):
-					os.mkdir(refFolder)
-				with open(refFolder + '/cppcheck-'+ variant + '.txt', 'w') as refFile:
-					refFile.write(vMsg)
 			vId += 1
 
 		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
@@ -213,7 +184,7 @@ class StaticCodeAnalysis():
 
 		return True
 
-	def LicenceAndFormattingCheck(self, HTML):
+	def LicenceAndFormattingCheck(self, ctx, HTML):
 		# Workspace is no longer recreated from scratch.
 		# It implies that this method shall be called last within a build pipeline
 		# where workspace is already created
@@ -238,7 +209,7 @@ class StaticCodeAnalysis():
 			else:
 				check_options += f' --build-arg TARGET_BRANCH={self.ranTargetBranch}'
 
-		logDir = f'{lSourcePath}/cmake_targets/build_log_{self.testCase_id}'
+		logDir = f'{lSourcePath}/cmake_targets/log/'
 		cmd.run(f'mkdir -p {logDir}')
 		cmd.run('docker image rm oai-formatting-check:latest')
 		cmd.run(f'docker build --target oai-formatting-check --tag oai-formatting-check:latest {check_options} --file {lSourcePath}/ci-scripts/docker/Dockerfile.formatting.bionic {lSourcePath} > {logDir}/oai-formatting-check.txt 2>&1')
@@ -247,12 +218,11 @@ class StaticCodeAnalysis():
 		cmd.run('docker image prune --force')
 		cmd.run('docker volume prune --force')
 
-		# Analyzing the logs
-		cmd.copyin(f'{logDir}/oai-formatting-check.txt', 'oai-formatting-check.txt')
+		file = archiveArtifact(cmd, ctx, f'{logDir}/oai-formatting-check.txt')
 		cmd.close()
 
 		finalStatus = 0
-		if (os.path.isfile('./oai-formatting-check.txt')):
+		if (os.path.isfile(file)):
 			analyzed = False
 			nbFilesNotFormatted = 0
 			listFiles = False
@@ -263,7 +233,7 @@ class StaticCodeAnalysis():
 			gnuGplLicenceFiles = []
 			suspectLicence = False
 			suspectLicenceFiles = []
-			with open('./oai-formatting-check.txt', 'r') as logfile:
+			with open(file, 'r') as logfile:
 				for line in logfile:
 					ret = re.search('./ci-scripts/checkCodingFormattingRules.sh', str(line))
 					if ret is not None:

ci-scripts/main.py

@@ -43,8 +43,10 @@ import cls_containerize	 #class Containerize for all container-based operations
 import cls_static_code_analysis  #class for static code analysis
 import cls_cluster		 # class for building/deploying on cluster
 import cls_native        # class for all native/source-based operations
+from cls_ci_helper import TestCaseCtx
 
 import ran
+import cls_cmd
 import cls_oai_html
 
 
@@ -91,7 +93,7 @@ def AssignParams(params_dict):
 		setattr(RAN, key, value)
 		setattr(HTML, key, value)
 
-def ExecuteActionWithParam(action):
+def ExecuteActionWithParam(action, ctx):
 	global RAN
 	global HTML
 	global CONTAINERS
@@ -134,15 +136,15 @@ def ExecuteActionWithParam(action):
 		if proxy_commit is not None:
 			CONTAINERS.proxyCommit = proxy_commit
 		if action == 'Build_eNB':
-			success = cls_native.Native.Build(HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args)
+			success = cls_native.Native.Build(ctx, HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args)
 		elif action == 'Build_Image':
-			success = CONTAINERS.BuildImage(HTML)
+			success = CONTAINERS.BuildImage(ctx, HTML)
 		elif action == 'Build_Proxy':
-			success = CONTAINERS.BuildProxy(HTML)
+			success = CONTAINERS.BuildProxy(ctx, HTML)
 		elif action == 'Build_Cluster_Image':
-			success = CLUSTER.BuildClusterImage(HTML)
+			success = CLUSTER.BuildClusterImage(ctx, HTML)
 		elif action == 'Build_Run_Tests':
-			success = CONTAINERS.BuildRunTests(HTML)
+			success = CONTAINERS.BuildRunTests(ctx, HTML)
 
 	elif action == 'Initialize_eNB':
 		datalog_rt_stats_file=test.findtext('rt_stats_cfg')
@@ -204,7 +206,7 @@ def ExecuteActionWithParam(action):
 			RAN.air_interface[RAN.eNB_instance] = 'lte-softmodem'
 		else:
 			RAN.air_interface[RAN.eNB_instance] = air_interface.lower() +'-softmodem'
-		success = RAN.TerminateeNB(HTML)
+		success = RAN.TerminateeNB(ctx, HTML)
 
 	elif action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Terminate_UE' or action == 'CheckStatusUE' or action == 'DataEnable_UE' or action == 'DataDisable_UE':
 		CiTestObj.ue_ids = test.findtext('id').split(' ')
@@ -226,7 +228,7 @@ def ExecuteActionWithParam(action):
 		elif action == 'Detach_UE':
 			success = CiTestObj.DetachUE(HTML)
 		elif action == 'Terminate_UE':
-			success = CiTestObj.TerminateUE(HTML)
+			success = CiTestObj.TerminateUE(ctx, HTML)
 		elif action == 'CheckStatusUE':
 			success = CiTestObj.CheckStatusUE(HTML)
 		elif action == 'DataEnable_UE':
@@ -253,7 +255,7 @@ def ExecuteActionWithParam(action):
 			else:
 				CiTestObj.nodes = [None] * len(CiTestObj.ue_ids)
 		ping_rttavg_threshold = test.findtext('ping_rttavg_threshold') or ''
-		success = CiTestObj.Ping(HTML, CONTAINERS)
+		success = CiTestObj.Ping(ctx, HTML)
 
 	elif action == 'Iperf' or action == 'Iperf2_Unidir':
 		CiTestObj.iperf_args = test.findtext('iperf_args')
@@ -284,9 +286,9 @@ def ExecuteActionWithParam(action):
 			logging.error('test-case has wrong option ' + CiTestObj.iperf_options)
 			CiTestObj.iperf_options = 'check'
 		if action == 'Iperf':
-			success = CiTestObj.Iperf(HTML, CONTAINERS)
+			success = CiTestObj.Iperf(ctx, HTML)
 		elif action == 'Iperf2_Unidir':
-			success = CiTestObj.Iperf2_Unidir(HTML, CONTAINERS)
+			success = CiTestObj.Iperf2_Unidir(ctx, HTML)
 
 	elif action == 'IdleSleep':
 		st = test.findtext('idle_sleep_time_in_sec') or "5"
@@ -295,12 +297,12 @@ def ExecuteActionWithParam(action):
 	elif action == 'Deploy_Run_PhySim':
 		oc_release = test.findtext('oc_release')
 		svr_id = test.findtext('svr_id') or None
-		success = CLUSTER.deploy_oc_physim(HTML, oc_release, svr_id)
+		success = CLUSTER.deploy_oc_physim(ctx, HTML, oc_release, svr_id)
 
 	elif action == 'DeployCoreNetwork' or action == 'UndeployCoreNetwork':
 		cn_id = test.findtext('cn_id')
 		core_op = getattr(cls_oaicitest.OaiCiTest, action)
-		success = core_op(cn_id, HTML)
+		success = core_op(cn_id, ctx, HTML)
 
 	elif action == 'Deploy_Object' or action == 'Undeploy_Object' or action == "Create_Workspace":
 		eNB_instance=test.findtext('eNB_instance')
@@ -328,9 +330,9 @@ def ExecuteActionWithParam(action):
 		CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1)
 		CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge)
 		if action == 'Deploy_Object':
-			success = CONTAINERS.DeployObject(HTML)
+			success = CONTAINERS.DeployObject(ctx, HTML)
 		elif action == 'Undeploy_Object':
-			success = CONTAINERS.UndeployObject(HTML, RAN)
+			success = CONTAINERS.UndeployObject(ctx, HTML, RAN)
 		elif action == 'Create_Workspace':
 			if force_local:
 				# Do not create a working directory when running locally. Current repo directory will be used
@@ -341,13 +343,13 @@ def ExecuteActionWithParam(action):
 		physim_options = test.findtext('physim_run_args')
 		physim_test = test.findtext('physim_test')
 		physim_threshold = test.findtext('physim_time_threshold') or 'inf'
-		success = cls_native.Native.Run_Physim(HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold)
+		success = cls_native.Native.Run_Physim(ctx, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold)
 
 	elif action == 'LicenceAndFormattingCheck':
-		success = SCA.LicenceAndFormattingCheck(HTML)
+		success = SCA.LicenceAndFormattingCheck(ctx, HTML)
 
 	elif action == 'Cppcheck_Analysis':
-		success = SCA.CppCheckAnalysis(HTML)
+		success = SCA.CppCheckAnalysis(ctx, HTML)
 
 	elif action == 'Push_Local_Registry':
 		svr_id = test.findtext('svr_id')
@@ -407,10 +409,11 @@ def test_in_list(test, list):
 def receive_signal(signum, frame):
 	sys.exit(1)
 
-
-
-
-
+def ShowTestID(ctx, desc):
+    logging.info(f'\u001B[1m----------------------------------------\u001B[0m')
+    logging.info(f'\u001B[1m Test ID: {ctx.test_id} (#{ctx.count}) \u001B[0m')
+    logging.info(f'\u001B[1m {desc} \u001B[0m')
+    logging.info(f'\u001B[1m----------------------------------------\u001B[0m')
 
 #-----------------------------------------------------------
 # MAIN PART
@@ -466,15 +469,7 @@ if py_param_file_present == True:
 cwd = os.getcwd()
 
 if re.match('^TerminateeNB$', mode, re.IGNORECASE):
-	if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '':
-		HELP.GenericHelp(CONST.Version)
-		sys.exit('Insufficient Parameter')
-	if RAN.eNBIPAddress == 'none':
-		sys.exit(0)
-	RAN.eNB_instance=0
-	RAN.eNB_serverId[0]='0'
-	RAN.eNBSourceCodePath='/tmp/'
-	RAN.TerminateeNB(HTML)
+	logging.warning("Option TerminateeNB ignored")
 elif re.match('^TerminateHSS$', mode, re.IGNORECASE):
 	logging.warning("Option TerminateHSS ignored")
 elif re.match('^TerminateMME$', mode, re.IGNORECASE):
@@ -482,12 +477,7 @@ elif re.match('^TerminateMME$', mode, re.IGNORECASE):
 elif re.match('^TerminateSPGW$', mode, re.IGNORECASE):
 	logging.warning("Option TerminateSPGW ignored")
 elif re.match('^LogCollectBuild$', mode, re.IGNORECASE):
-	if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '':
-		logging.warning("nothing to collect (eNBIPAddress/eNBUserName/eNBPassword/eNBSourceCodePath is '')")
-		sys.exit(0)
-	if RAN.eNBIPAddress == 'none':
-		sys.exit(0)
-	CiTestObj.LogCollectBuild(RAN)
+	logging.warning("Option LogCollectBuild ignored")
 elif re.match('^LogCollecteNB$', mode, re.IGNORECASE):
 	if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '':
 		HELP.GenericHelp(CONST.Version)
@@ -565,6 +555,16 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 	else:
 		xml_test_file = cwd + "/" + CiTestObj.testXMLfiles[0]
 
+	# directory where all log artifacts will be placed
+	logPath = f"{cwd}/../cmake_targets/log/{CiTestObj.testXMLfiles[0].split('/')[-1]}.d"
+	# we run from within ci-scripts, but the logPath is absolute, so replace
+	# the ci-scripts/..; if it does not exist, nothing will happen
+	logPath = logPath.replace(r'/ci-scripts/..', '')
+	logging.info(f"placing all artifacts for this run in {logPath}/")
+	with cls_cmd.LocalCmd() as c:
+		c.run(f"rm -rf {logPath}")
+		c.run(f"mkdir -p {logPath}")
+
 	xmlTree = ET.parse(xml_test_file)
 	xmlRoot = xmlTree.getroot()
 
@@ -615,29 +615,32 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 	task_set_succeeded = True
 	HTML.startTime=int(round(time.time() * 1000))
 
+	i = 0
 	for test_case_id in todo_tests:
 		for test in all_tests:
 			id = test.get('id')
 			if test_case_id != id:
 				continue
+			i += 1
 			CiTestObj.testCase_id = id
+			ctx = TestCaseCtx(i, int(id), logPath)
 			HTML.testCase_id=CiTestObj.testCase_id
-			CiTestObj.desc = test.findtext('desc')
+			desc = test.findtext('desc')
 			always_exec = test.findtext('always_exec') in ['True', 'true', 'Yes', 'yes']
 			may_fail = test.findtext('may_fail') in ['True', 'true', 'Yes', 'yes']
-			HTML.desc=CiTestObj.desc
+			HTML.desc = desc
 			action = test.findtext('class')
 			if (CheckClassValidity(xml_class_list, action, id) == False):
 				task_set_succeeded = False
 				continue
-			CiTestObj.ShowTestID()
+			ShowTestID(ctx, desc)
 			if not task_set_succeeded and not always_exec:
 				msg = f"skipping test due to prior error"
 				logging.warning(msg)
 				HTML.CreateHtmlTestRowQueue(msg, "SKIP", [])
 				break
 			try:
-				test_succeeded = ExecuteActionWithParam(action)
+				test_succeeded = ExecuteActionWithParam(action, ctx)
 				if not test_succeeded and may_fail:
 					logging.warning(f"test ID {test_case_id} action {action} may or may not fail, proceeding despite error")
 				elif not test_succeeded:

ci-scripts/ran.py

@@ -46,6 +46,7 @@ import cls_cmd
 import sshconnection as SSH
 import helpreadme as HELP
 import constants as CONST
+from cls_ci_helper import archiveArtifact
 
 #-----------------------------------------------------------
 # Class Declaration
@@ -124,7 +125,6 @@ class RANManagement():
 
 		self.testCase_id = HTML.testCase_id
 		mySSH = SSH.SSHConnection()
-		cwd = os.getcwd()
 		
 		mySSH.open(lIpAddr, lUserName, lPassWord)
 		mySSH.command('cd ' + lSourcePath, '\$', 5)
@@ -249,42 +249,7 @@ class RANManagement():
 		logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m')
 		return enbDidSync
 
-	def CheckeNBProcess(self, status_queue):
-		# At least the instance 0 SHALL be on!
-		if self.eNBstatuses[0] == 0:
-			lIpAddr = self.eNBIPAddress
-			lUserName = self.eNBUserName
-			lPassWord = self.eNBPassword
-		elif self.eNBstatuses[0] == 1:
-			lIpAddr = self.eNB1IPAddress
-			lUserName = self.eNB1UserName
-			lPassWord = self.eNB1Password
-		elif self.eNBstatuses[0] == 2:
-			lIpAddr = self.eNB2IPAddress
-			lUserName = self.eNB2UserName
-			lPassWord = self.eNB2Password
-		else:
-			lIpAddr = self.eNBIPAddress
-			lUserName = self.eNBUserName
-			lPassWord = self.eNBPassword
-		mySSH = SSH.SSHConnection()
-		mySSH.open(lIpAddr, lUserName, lPassWord)
-		if self.air_interface[self.eNB_instance] == '':
-			pattern = 'softmodem'
-		else:
-			pattern = self.air_interface[self.eNB_instance]
-		mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + pattern + ' | grep -v grep', '\$', 5)
-		result = re.search(pattern, mySSH.getBefore())
-		success = result is not None
-		if not success:
-			logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
-			status_queue.put(CONST.ENB_PROCESS_FAILED)
-		else:
-			status_queue.put(CONST.ENB_PROCESS_OK)
-		mySSH.close()
-		return success
-
-	def TerminateeNB(self, HTML):
+	def TerminateeNB(self, ctx, HTML):
 		if self.eNB_serverId[self.eNB_instance] == '0':
 			lIpAddr = self.eNBIPAddress
 			lUserName = self.eNBUserName
@@ -356,13 +321,11 @@ class RANManagement():
 				fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
 				self.eNBLogFiles[int(self.eNB_instance)] = ''
 			if analyzeFile:
-				#*stats.log files + pickle + png
-				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*stats.log', '.')
-				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.pickle', '.')
-				mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.png', '.')
-				#
-				copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
-				if (copyin_res == -1):
+				with cls_cmd.getConnection(lIpAddr) as cmd:
+					file = archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/' + fileToAnalyze)
+					archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrL1_stats.log')
+					archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrMAC_stats.log')
+				if file is None:
 					logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
 					HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
 					HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
@@ -375,8 +338,8 @@ class RANManagement():
 					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrL1_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
 					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrMAC_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
 					mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
-				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze)
-				logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML, self.ran_checkers)
+				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + file)
+				logStatus = self.AnalyzeLogFile_eNB(file, HTML, self.ran_checkers)
 				if (logStatus < 0):
 					HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
 					#display rt stats for gNB only
@@ -734,11 +697,16 @@ class RANManagement():
 			datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader)
 		rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field  
 
-		if os.path.isfile('./nrL1_stats.log') and os.path.isfile('./nrMAC_stats.log'):
+		# nrL1_stats.log/nrMAC_stats.log should be in the same directory as main log file
+		# currently the link is only implicit as below based on pattern matching
+		# I will rework this to give the file explicitly
+		l1_stats_fn = re.sub(r'enb_[0-9]+', 'nrL1_stats', eNBlogFile)
+		mac_stats_fn = re.sub(r'enb_[0-9]+', 'nrMAC_stats', eNBlogFile)
+		if os.path.isfile(l1_stats_fn) and os.path.isfile(mac_stats_fn):
 			# don't use CI-nrL1_stats.log, as this will increase the processing time for
 			# no reason, we just need the last occurence
-			nrL1_stats = open('./nrL1_stats.log', 'r')
-			nrMAC_stats = open('./nrMAC_stats.log', 'r')
+			nrL1_stats = open(l1_stats_fn, 'r')
+			nrMAC_stats = open(mac_stats_fn, 'r')
 			for line in nrL1_stats.readlines():
 				for k in rt_keys:
 					result = re.search(k, line)     
@@ -758,7 +726,7 @@ class RANManagement():
 			nrL1_stats.close()
 			nrMAC_stats.close()
 		else:
-			logging.debug("NR Stats files for RT analysis not found")
+			logging.debug(f"NR Stats files for RT analysis not found: {l1_stats_fn}, {mac_stats_fn}")
 
 		#stdout log file and stat log files analysis completed
 		logging.debug('   File analysis (stdout, stats) completed')

ci-scripts/tests/config/test_module_infra.yaml

+test:
+  Host: localhost
+  InitScript: ""
+  TermScript: ""
+  AttachScript: "true"
+  DetachScript: "true"
+  NetworkScript: ip a show dev lo
+  IF: lo
+  MTU: 65536
+
+test-fail:
+  Host: localhost
+  InitScript: ""
+  TermScript: ""
+  AttachScript: "false"
+  DetachScript: "true"
+  NetworkScript: ip a show dev TESTDEVICEDOESNOTEXIST
+  IF: TESTDEVICEDOESNOTEXIST
+  MTU: 123
+
+test-trace:
+  Host: localhost
+  InitScript: ""
+  TermScript: ""
+  AttachScript: "true"
+  DetachScript: "true"
+  Tracing:
+    # tttf: temp-trace-test-file
+    Start: "rm -f /tmp/tttf; for i in {1..8}; do echo $i >> /tmp/tttf; sleep 1; done &"
+    Stop: "while [[ $(wc -l /tmp/tttf) < 4 ]]; do sleep 1; done"
+    Collect: "cp /tmp/tttf %%log_dir%%"
+  NetworkScript: ip a show dev lo
+  IF: lo
+  MTU: 65536

ci-scripts/tests/corenetwork.py

@@ -6,14 +6,22 @@ logging.basicConfig(
 	format="[%(asctime)s] %(levelname)8s: %(message)s"
 )
 import os
+import tempfile
 
 import unittest
 
 sys.path.append('./') # to find OAI imports below
 import cls_corenetwork
+from cls_ci_helper import TestCaseCtx
 import cls_cmd
 
 class TestCoreNetwork(unittest.TestCase):
+    def setUp(self):
+        self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
+    def tearDown(self):
+        with cls_cmd.LocalCmd() as c:
+            c.run(f'rm -rf {self.ctx.logPath}')
+
     def test_simple_core(self):
         c = cls_corenetwork.CoreNetwork("test", filename="tests/config/test_core_infra.yaml")
         success, output = c.deploy()
@@ -22,14 +30,14 @@ class TestCoreNetwork(unittest.TestCase):
         # should not have it set in config
         self.assertTrue(c.runIperf3Server())
         self.assertEqual(c.getIP(), "127.0.0.1")
-        log_files, output = c.undeploy(log_dir="/tmp")
+        log_files, output = c.undeploy(self.ctx)
         self.assertEqual(output, "undeploy")
         with cls_cmd.LocalCmd() as cmd:
             # there must be one log file for this (test) core
             self.assertEqual(len(log_files), 1)
-            # test core writes to %%log_dir%%/logs, which expands to /tmp/logs
+            # undeploy uses archiveArtifact(), which writes to {prefix}-logs
             l = log_files[0]
-            self.assertEqual(l, "/tmp/logs")
+            self.assertEqual(l, f"{self.ctx.baseFilename()}-logs")
             ret = cmd.run(f"cat {l}")
             self.assertEqual(ret.returncode, 0) # command must succeed
             self.assertEqual(ret.stdout, "logs") # output should be "logs"
@@ -39,14 +47,14 @@ class TestCoreNetwork(unittest.TestCase):
         success, _ = c.deploy()
         self.assertTrue(success)
         self.assertFalse(c.runIperf3Server())
-        c.undeploy(log_dir="/tmp")
+        c.undeploy(None)
 
     def test_core_fail(self):
         c = cls_corenetwork.CoreNetwork("test_fail", filename="tests/config/test_core_infra.yaml")
         success, _ = c.deploy()
         self.assertFalse(success)
         # undeployment should still work
-        c.undeploy(log_dir="/tmp")
+        c.undeploy(None)
 
     def test_core_script(self):
         c = cls_corenetwork.CoreNetwork("test_script", filename="tests/config/test_core_infra.yaml")

ci-scripts/tests/deployment.py

@@ -6,6 +6,7 @@ logging.basicConfig(
 	format="[%(asctime)s] %(levelname)8s: %(message)s"
 )
 import os
+import tempfile
 os.system(f'rm -rf cmake_targets')
 os.system(f'mkdir -p cmake_targets/log')
 import unittest
@@ -14,6 +15,7 @@ sys.path.append('./') # to find OAI imports below
 import cls_oai_html
 import cls_oaicitest
 import cls_containerize
+from cls_ci_helper import TestCaseCtx
 import ran
 import cls_cmd
 
@@ -48,12 +50,16 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.eNBPassword = None
 		self.cont.eNBSourceCodePath = os.getcwd()
 		self.cont.num_attempts = 3
+		self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
+	def tearDown(self):
+		with cls_cmd.LocalCmd() as c:
+			c.run(f'rm -rf {self.ctx.logPath}')
 
 	def test_deploy(self):
 		self.cont.yamlPath[0] = 'tests/simple-dep/'
 		self.cont.deploymentTag = "noble"
-		deploy = self.cont.DeployObject(self.html)
-		undeploy = self.cont.UndeployObject(self.html, self.ran)
+		deploy = self.cont.DeployObject(self.ctx, self.html)
+		undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertTrue(deploy)
 		self.assertTrue(undeploy)
 
@@ -61,8 +67,8 @@ class TestDeploymentMethods(unittest.TestCase):
 		# fails reliably
 		old = self.cont.yamlPath
 		self.cont.yamlPath[0] = 'tests/simple-fail/'
-		deploy = self.cont.DeployObject(self.html)
-		self.cont.UndeployObject(self.html, self.ran)
+		deploy = self.cont.DeployObject(self.ctx, self.html)
+		self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertFalse(deploy)
 		self.cont.yamlPath = old
 
@@ -70,8 +76,8 @@ class TestDeploymentMethods(unittest.TestCase):
 		# fails reliably
 		old = self.cont.yamlPath
 		self.cont.yamlPath[0] = 'tests/simple-fail-2svc/'
-		deploy = self.cont.DeployObject(self.html)
-		self.cont.UndeployObject(self.html, self.ran)
+		deploy = self.cont.DeployObject(self.ctx, self.html)
+		self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertFalse(deploy)
 		self.cont.yamlPath = old
 
@@ -79,8 +85,8 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
 		self.cont.services[0] = "oai-gnb"
 		self.cont.deploymentTag = 'develop-12345678'
-		deploy = self.cont.DeployObject(self.html)
-		undeploy = self.cont.UndeployObject(self.html, self.ran)
+		deploy = self.cont.DeployObject(self.ctx, self.html)
+		undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertTrue(deploy)
 		self.assertTrue(undeploy)
 
@@ -88,8 +94,8 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
 		self.cont.services[0] = "oai-gnb oai-nr-ue"
 		self.cont.deploymentTag = 'develop-12345678'
-		deploy = self.cont.DeployObject(self.html)
-		undeploy = self.cont.UndeployObject(self.html, self.ran)
+		deploy = self.cont.DeployObject(self.ctx, self.html)
+		undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertTrue(deploy)
 		self.assertTrue(undeploy)
 
@@ -97,10 +103,10 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
 		self.cont.services[0] = "oai-gnb"
 		self.cont.deploymentTag = 'develop-12345678'
-		deploy1 = self.cont.DeployObject(self.html)
+		deploy1 = self.cont.DeployObject(self.ctx, self.html)
 		self.cont.services[0] = "oai-nr-ue"
-		deploy2 = self.cont.DeployObject(self.html)
-		undeploy = self.cont.UndeployObject(self.html, self.ran)
+		deploy2 = self.cont.DeployObject(self.ctx, self.html)
+		undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
 		self.assertTrue(deploy1)
 		self.assertTrue(deploy2)
 		self.assertTrue(undeploy)

ci-scripts/tests/module.py

+import sys
+import logging
+import tempfile
+logging.basicConfig(
+	level=logging.DEBUG,
+	stream=sys.stdout,
+	format="[%(asctime)s] %(levelname)8s: %(message)s"
+)
+import os
+
+import unittest
+
+sys.path.append('./') # to find OAI imports below
+import cls_module
+from cls_ci_helper import TestCaseCtx
+import cls_cmd
+
+class TestModule(unittest.TestCase):
+    def setUp(self):
+        self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
+    def tearDown(self):
+        with cls_cmd.LocalCmd() as c:
+            c.run(f'rm -rf {self.ctx.logPath}')
+
+    def test_simple_module(self):
+        c = cls_module.Module_UE("test", filename="tests/config/test_module_infra.yaml")
+        self.assertFalse(c.trace)
+        success = c.initialize()
+        self.assertTrue(success)
+        ip = c.attach()
+        self.assertEqual(ip, "127.0.0.1")
+        self.assertTrue(c.checkMTU())
+        c.detach()
+        logs = c.terminate()
+        self.assertEqual(logs, None) # no tracing
+
+    @unittest.skip("this test takes long: it verifies the UE cannot attach")
+    def test_simple_fail(self):
+        c = cls_module.Module_UE("test-fail", filename="tests/config/test_module_infra.yaml")
+        success = c.initialize()
+        self.assertTrue(success)
+        ip = c.attach()
+        self.assertEqual(ip, None)
+        self.assertFalse(c.checkMTU())
+        c.detach()
+        logs = c.terminate()
+        self.assertEqual(logs, None) # no tracing
+
+    def test_simple_trace(self):
+        c = cls_module.Module_UE("test-trace", filename="tests/config/test_module_infra.yaml")
+        self.assertTrue(c.trace)
+        success = c.initialize()
+        self.assertTrue(success)
+        ip = c.attach()
+        self.assertEqual(ip, "127.0.0.1")
+        self.assertTrue(c.checkMTU())
+        c.detach()
+        tmp = tempfile.mkdtemp()
+        log_file = c.terminate(self.ctx)
+        # undeploy uses archiveArtifact(), which writes to {prefix}-logs
+        self.assertEqual(log_file, [f"{self.ctx.baseFilename()}-tttf"]) # matches test-trace UE collection file
+
+if __name__ == '__main__':
+    unittest.main()

ci-scripts/tests/ping-iperf.py

@@ -9,11 +9,13 @@ import os
 os.system(f'rm -rf cmake_targets')
 os.system(f'mkdir -p cmake_targets/log')
 import unittest
+import tempfile
 
 sys.path.append('./') # to find OAI imports below
 import cls_oai_html
+from cls_ci_helper import TestCaseCtx
 import cls_oaicitest
-import cls_containerize
+import cls_cmd
 
 class TestPingIperf(unittest.TestCase):
 	def setUp(self):
@@ -22,7 +24,10 @@ class TestPingIperf(unittest.TestCase):
 		self.ci = cls_oaicitest.OaiCiTest()
 		self.ci.ue_ids = ["test"]
 		self.ci.nodes = ["localhost"]
-		self.cont = cls_containerize.Containerize()
+		self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
+	def tearDown(self):
+		with cls_cmd.LocalCmd() as c:
+			c.run(f'rm -rf {self.ctx.logPath}')
 
 	def test_ping(self):
 		self.ci.ping_args = "-c3"
@@ -30,7 +35,7 @@ class TestPingIperf(unittest.TestCase):
 		self.ci.svr_id = "test"
 		infra_file = "tests/config/infra_ping_iperf.yaml"
 		# TODO Should need nothing but options and UE(s) to use
-		success = self.ci.Ping(self.html, self.cont, infra_file=infra_file)
+		success = self.ci.Ping(self.ctx, self.html, infra_file=infra_file)
 		self.assertTrue(success)
 
 	def test_iperf(self):
@@ -45,7 +50,7 @@ class TestPingIperf(unittest.TestCase):
 		self.ci.iperf_profile = "balanced"
 		infra_file = "tests/config/infra_ping_iperf.yaml"
 		# TODO Should need nothing but options and UE(s) to use
-		success = self.ci.Iperf(self.html, self.cont, infra_file=infra_file)
+		success = self.ci.Iperf(self.ctx, self.html, infra_file=infra_file)
 		self.assertTrue(success)
 
 	def test_iperf2_unidir(self):
@@ -57,7 +62,7 @@ class TestPingIperf(unittest.TestCase):
 		self.ci.iperf_profile = "balanced"
 		infra_file = "tests/config/infra_ping_iperf.yaml"
 		# TODO Should need nothing but options and UE(s) to use
-		success = self.ci.Iperf2_Unidir(self.html, self.cont, infra_file=infra_file)
+		success = self.ci.Iperf2_Unidir(self.ctx, self.html, infra_file=infra_file)
 		self.assertTrue(success)
 
 	def test_iperf_highrate(self):

ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml

@@ -26,7 +26,6 @@
         <htmlTabIcon>wrench</htmlTabIcon>
         <TestCaseRequestedList>
  111111
- 100001
  800813
  000001
  000002

ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml

@@ -26,7 +26,6 @@
         <htmlTabIcon>wrench</htmlTabIcon>
         <TestCaseRequestedList>
  111111
- 100001
  800813
  000001
  000002

ci-scripts/xml_files/gnb_usrp_build.xml

@@ -47,6 +47,7 @@
 	<testCase id="100000">
 		<class>Custom_Command</class>
 		<desc>Reboot USRP</desc>
+		<may_fail>true</may_fail>
 		<node>caracal</node>
 		<command>ssh root@192.168.20.2 reboot</command>
         </testCase>

docker/Dockerfile.build.fhi72.native_arm.ubuntu22

@@ -78,7 +78,7 @@ RUN /bin/sh oaienv && \
     cd cmake_targets && \
     mkdir -p log && \
     ./build_oai \
-      --gNB \
+      --ninja --gNB \
       --build-lib "telnetsrv enbscope uescope nrscope" \
       -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
       --build-e2 --cmake-opt -DXAPP_MULTILANGUAGE=OFF --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \

docker/Dockerfile.build.fhi72.rhel9

@@ -58,7 +58,7 @@ RUN /bin/sh oaienv && \
     mkdir -p log && \
     scl enable gcc-toolset-13 "\
     ./build_oai \
-      --gNB \
+      --ninja --gNB \
       --build-lib 'telnetsrv enbscope uescope nrscope' \
       -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
       --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION" &&\

docker/Dockerfile.build.fhi72.ubuntu22

@@ -68,7 +68,7 @@ RUN /bin/sh oaienv && \
     cd cmake_targets && \
     mkdir -p log && \
     ./build_oai \
-      --gNB \
+      --ninja --gNB \
       --build-lib "telnetsrv enbscope uescope nrscope" \
       -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
       --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \