Commit 429e98e4 authored by Jaroslava Fiedlerova's avatar Jaroslava Fiedlerova

Merge remote-tracking branch 'origin/ci-container-restart' into integration_2024_w49 (!3141)

CI: Allow restart of the container if deployment fails

In CI we sometimes encounter fail of the gNB/UE deployment (and fail of the test
scenario) caused by unsuccessful initialization of the USRP N310.

This MR enables to optionally restart the gNB or UE container, if the deployment
fails on the health check during start up. By default, restarts of the container
are not allowed, but we can enable them by setting num_attemps > 1 for a given
deployment in the XML file.

Logs from failed deployment attempts are collected.

This MR aims to avoid known CI failure caused by "USRP N310 Initialization Failure"
mentioned in #871.
parents 8c5b37de 027631eb
......@@ -145,10 +145,6 @@ def AnalyzeBuildLogs(buildRoot, images, globalStatus):
collectInfo[image] = files
return collectInfo
def GetContainerName(ssh, svcName, file):
ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".container_name'", silent=True)
return ret.stdout
def GetImageName(ssh, svcName, file):
ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".image'", silent=True)
if ret.returncode != 0:
......@@ -156,17 +152,18 @@ def GetImageName(ssh, svcName, file):
else:
return ret.stdout.strip()
def GetContainerHealth(ssh, containerName):
if containerName is None:
return False
if 'db_init' in containerName or 'db-init' in containerName: # exits with 0, there cannot be healthy
return True
def GetServiceHealth(ssh, svcName, file):
if svcName is None:
return False, f"Service {svcName} not found in {file}"
image = GetImageName(ssh, svcName, file)
if 'db_init' in svcName or 'db-init' in svcName: # exits with 0, there cannot be healthy
return True, f"Service {svcName} healthy, image {image}"
for _ in range(8):
result = ssh.run(f'docker inspect --format="{{{{.State.Health.Status}}}}" {containerName}', silent=True)
result = ssh.run(f"docker compose -f {file} ps --format json {svcName} | jq -r 'if type==\"array\" then .[0].Health else .Health end'", silent=True)
if result.stdout == 'healthy':
return True
return True, f"Service {svcName} healthy, image {image}"
time.sleep(5)
return False
return False, f"Failed to deploy: service {svcName}"
def ExistEnvFilePrint(ssh, wd, prompt='env vars in existing'):
ret = ssh.run(f'cat {wd}/.env', silent=True, reportNonZero=False)
......@@ -218,9 +215,9 @@ def GetServices(ssh, requested, file):
else:
return requested
def CopyinContainerLog(ssh, lSourcePath, yaml, containerName, filename):
def CopyinServiceLog(ssh, lSourcePath, yaml, svcName, wd_yaml, filename):
remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}"
ssh.run(f'docker logs {containerName} &> {remote_filename}')
ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}')
local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}"
local_filename = f"{local_dir}/{filename}"
return ssh.copyin(remote_filename, local_filename)
......@@ -336,6 +333,7 @@ class Containerize():
self.imageToCopy = ''
#checkers from xml
self.ran_checkers={}
self.num_attempts = 1
self.flexricTag = ''
......@@ -843,50 +841,55 @@ class Containerize():
def DeployObject(self, HTML):
svr = self.eNB_serverId[self.eNB_instance]
num_attempts = self.num_attempts
lIpAddr, lSourcePath = self.GetCredentials(svr)
logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m')
logging.debug(f'Deploying OAI Object on server: {lIpAddr}')
yaml = self.yamlPath[self.eNB_instance].strip('/')
# creating the log folder by default
local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}"
os.system(f'mkdir -p {local_dir}')
wd = f'{lSourcePath}/{yaml}'
wd_yaml = f'{wd}/docker-compose.y*ml'
yaml_dir = yaml.split('/')[-1]
with cls_cmd.getConnection(lIpAddr) as ssh:
services = GetServices(ssh, self.services[self.eNB_instance], f"{wd}/docker-compose.y*ml")
services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml)
if services == [] or services == ' ' or services == None:
msg = "Cannot determine services to start"
msg = 'Cannot determine services to start'
logging.error(msg)
HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg])
return False
ExistEnvFilePrint(ssh, wd)
WriteEnvFile(ssh, services, wd, self.deploymentTag, self.flexricTag)
logging.info(f"will start services {services}")
status = ssh.run(f'docker compose -f {wd}/docker-compose.y*ml up -d -- {services}')
if status.returncode != 0:
msg = f"cannot deploy services {services}: {status.stdout}"
logging.error(msg)
HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg])
return False
imagesInfo = []
fstatus = True
for svc in services.split():
containerName = GetContainerName(ssh, svc, f"{wd}/docker-compose.y*ml")
healthy = GetContainerHealth(ssh, containerName)
if not healthy:
imagesInfo += [f"Failed to deploy: service {svc}"]
fstatus = False
else:
image = GetImageName(ssh, svc, f"{wd}/docker-compose.y*ml")
logging.info(f"service {svc} healthy, container {containerName}, image {image}")
imagesInfo += [f"service {svc} healthy, image {image}"]
if fstatus:
if num_attempts <= 0:
raise ValueError(f'Invalid value for num_attempts: {num_attempts}, must be greater than 0')
for attempt in range(num_attempts):
imagesInfo = []
healthInfo = []
logging.info(f'will start services {services}')
status = ssh.run(f'docker compose -f {wd_yaml} up -d -- {services}')
if status.returncode != 0:
msg = f'cannot deploy services {services}: {status.stdout}'
logging.error(msg)
HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [msg])
return False
for svc in services.split():
health, msg = GetServiceHealth(ssh, svc, f'{wd_yaml}')
logging.info(msg)
imagesInfo.append(msg)
healthInfo.append(health)
deployed = all(healthInfo)
if deployed:
break
elif (attempt < num_attempts - 1):
logging.warning(f'Failed to deploy on attempt {attempt}, restart services {services}')
for svc in services.split():
CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log')
ssh.run(f'docker compose -f {wd_yaml} down -- {services}')
if deployed:
HTML.CreateHtmlTestRowQueue('N/A', 'OK', ['\n'.join(imagesInfo)])
else:
HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)])
return fstatus
return deployed
def UndeployObject(self, HTML, RAN):
svr = self.eNB_serverId[self.eNB_instance]
......@@ -902,7 +905,7 @@ class Containerize():
if services is not None:
all_serv = " ".join([s for s, _ in services])
ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}')
copyin_res = all(CopyinContainerLog(ssh, lSourcePath, yaml_dir, c, f'{s}-{HTML.testCase_id}.log') for s, c in services)
copyin_res = all(CopyinServiceLog(ssh, lSourcePath, yaml_dir, s, f"{wd}/docker-compose.y*ml", f'{s}-{HTML.testCase_id}.log') for s, c in services)
else:
logging.warning('could not identify services to stop => no log file')
ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v')
......
......@@ -349,6 +349,7 @@ def ExecuteActionWithParam(action):
string_field = test.findtext('services')
if string_field is not None:
CONTAINERS.services[CONTAINERS.eNB_instance] = string_field
CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1)
CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge)
if action == 'Deploy_Object':
success = CONTAINERS.DeployObject(HTML)
......
......@@ -47,6 +47,7 @@ class TestDeploymentMethods(unittest.TestCase):
self.cont.eNBUserName = None
self.cont.eNBPassword = None
self.cont.eNBSourceCodePath = os.getcwd()
self.cont.num_attempts = 3
def test_deploy(self):
self.cont.yamlPath[0] = 'tests/simple-dep/'
......@@ -65,6 +66,15 @@ class TestDeploymentMethods(unittest.TestCase):
self.assertFalse(deploy)
self.cont.yamlPath = old
def test_deployfails_2svc(self):
# fails reliably
old = self.cont.yamlPath
self.cont.yamlPath[0] = 'tests/simple-fail-2svc/'
deploy = self.cont.DeployObject(self.html)
self.cont.UndeployObject(self.html, self.ran)
self.assertFalse(deploy)
self.cont.yamlPath = old
def test_deploy_ran(self):
self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
self.cont.services[0] = "oai-gnb"
......
services:
test1:
image: ubuntu:jammy
container_name: test_container1
cap_drop:
- ALL
entrypoint: /bin/bash -c "false"
healthcheck:
test: /bin/bash -c "true"
interval: 1s
timeout: 1s
retries: 1
test2:
image: ubuntu:jammy
container_name: test_container2
cap_drop:
- ALL
entrypoint: /bin/bash -c "sleep infinity"
healthcheck:
test: /bin/bash -c "true"
interval: 1s
timeout: 1s
retries: 1
......@@ -89,6 +89,7 @@
<yaml_path>ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz</yaml_path>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
<num_attempts>3</num_attempts>
</testCase>
<testCase id="000001">
......
......@@ -89,6 +89,7 @@
<yaml_path>ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz</yaml_path>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
<num_attempts>3</num_attempts>
</testCase>
<testCase id="000001">
......
......@@ -89,6 +89,7 @@
<yaml_path>ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz</yaml_path>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
<num_attempts>3</num_attempts>
</testCase>
<testCase id="000001">
......
......@@ -80,6 +80,7 @@
<yaml_path>ci-scripts/yaml_files/5g_sa_n310_gnb</yaml_path>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
<num_attempts>3</num_attempts>
</testCase>
<testCase id="800814">
<class>Create_Workspace</class>
......@@ -93,6 +94,7 @@
<yaml_path>ci-scripts/yaml_files/5g_sa_n310_nrue</yaml_path>
<eNB_instance>1</eNB_instance>
<eNB_serverId>1</eNB_serverId>
<num_attempts>3</num_attempts>
</testCase>
<testCase id="000001">
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment