Commit 429e98e4 authored by Jaroslava Fiedlerova's avatar Jaroslava Fiedlerova

Merge remote-tracking branch 'origin/ci-container-restart' into integration_2024_w49 (!3141)

CI: Allow restart of the container if deployment fails

In CI we sometimes encounter fail of the gNB/UE deployment (and fail of the test
scenario) caused by unsuccessful initialization of the USRP N310.

This MR enables to optionally restart the gNB or UE container, if the deployment
fails on the health check during start up. By default, restarts of the container
are not allowed, but we can enable them by setting num_attemps > 1 for a given
deployment in the XML file.

Logs from failed deployment attempts are collected.

This MR aims to avoid known CI failure caused by "USRP N310 Initialization Failure"
mentioned in #871.
parents 8c5b37de 027631eb
...@@ -145,10 +145,6 @@ def AnalyzeBuildLogs(buildRoot, images, globalStatus): ...@@ -145,10 +145,6 @@ def AnalyzeBuildLogs(buildRoot, images, globalStatus):
collectInfo[image] = files collectInfo[image] = files
return collectInfo return collectInfo
def GetContainerName(ssh, svcName, file):
ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".container_name'", silent=True)
return ret.stdout
def GetImageName(ssh, svcName, file): def GetImageName(ssh, svcName, file):
ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".image'", silent=True) ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".image'", silent=True)
if ret.returncode != 0: if ret.returncode != 0:
...@@ -156,17 +152,18 @@ def GetImageName(ssh, svcName, file): ...@@ -156,17 +152,18 @@ def GetImageName(ssh, svcName, file):
else: else:
return ret.stdout.strip() return ret.stdout.strip()
def GetContainerHealth(ssh, containerName): def GetServiceHealth(ssh, svcName, file):
if containerName is None: if svcName is None:
return False return False, f"Service {svcName} not found in {file}"
if 'db_init' in containerName or 'db-init' in containerName: # exits with 0, there cannot be healthy image = GetImageName(ssh, svcName, file)
return True if 'db_init' in svcName or 'db-init' in svcName: # exits with 0, there cannot be healthy
return True, f"Service {svcName} healthy, image {image}"
for _ in range(8): for _ in range(8):
result = ssh.run(f'docker inspect --format="{{{{.State.Health.Status}}}}" {containerName}', silent=True) result = ssh.run(f"docker compose -f {file} ps --format json {svcName} | jq -r 'if type==\"array\" then .[0].Health else .Health end'", silent=True)
if result.stdout == 'healthy': if result.stdout == 'healthy':
return True return True, f"Service {svcName} healthy, image {image}"
time.sleep(5) time.sleep(5)
return False return False, f"Failed to deploy: service {svcName}"
def ExistEnvFilePrint(ssh, wd, prompt='env vars in existing'): def ExistEnvFilePrint(ssh, wd, prompt='env vars in existing'):
ret = ssh.run(f'cat {wd}/.env', silent=True, reportNonZero=False) ret = ssh.run(f'cat {wd}/.env', silent=True, reportNonZero=False)
...@@ -218,9 +215,9 @@ def GetServices(ssh, requested, file): ...@@ -218,9 +215,9 @@ def GetServices(ssh, requested, file):
else: else:
return requested return requested
def CopyinContainerLog(ssh, lSourcePath, yaml, containerName, filename): def CopyinServiceLog(ssh, lSourcePath, yaml, svcName, wd_yaml, filename):
remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}" remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}"
ssh.run(f'docker logs {containerName} &> {remote_filename}') ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}')
local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}" local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}"
local_filename = f"{local_dir}/{filename}" local_filename = f"{local_dir}/{filename}"
return ssh.copyin(remote_filename, local_filename) return ssh.copyin(remote_filename, local_filename)
...@@ -336,6 +333,7 @@ class Containerize(): ...@@ -336,6 +333,7 @@ class Containerize():
self.imageToCopy = '' self.imageToCopy = ''
#checkers from xml #checkers from xml
self.ran_checkers={} self.ran_checkers={}
self.num_attempts = 1
self.flexricTag = '' self.flexricTag = ''
...@@ -843,50 +841,55 @@ class Containerize(): ...@@ -843,50 +841,55 @@ class Containerize():
def DeployObject(self, HTML): def DeployObject(self, HTML):
svr = self.eNB_serverId[self.eNB_instance] svr = self.eNB_serverId[self.eNB_instance]
num_attempts = self.num_attempts
lIpAddr, lSourcePath = self.GetCredentials(svr) lIpAddr, lSourcePath = self.GetCredentials(svr)
logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m') logging.debug(f'Deploying OAI Object on server: {lIpAddr}')
yaml = self.yamlPath[self.eNB_instance].strip('/') yaml = self.yamlPath[self.eNB_instance].strip('/')
# creating the log folder by default # creating the log folder by default
local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}" local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}"
os.system(f'mkdir -p {local_dir}') os.system(f'mkdir -p {local_dir}')
wd = f'{lSourcePath}/{yaml}' wd = f'{lSourcePath}/{yaml}'
wd_yaml = f'{wd}/docker-compose.y*ml'
yaml_dir = yaml.split('/')[-1]
with cls_cmd.getConnection(lIpAddr) as ssh: with cls_cmd.getConnection(lIpAddr) as ssh:
services = GetServices(ssh, self.services[self.eNB_instance], f"{wd}/docker-compose.y*ml") services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml)
if services == [] or services == ' ' or services == None: if services == [] or services == ' ' or services == None:
msg = "Cannot determine services to start" msg = 'Cannot determine services to start'
logging.error(msg) logging.error(msg)
HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg]) HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg])
return False return False
ExistEnvFilePrint(ssh, wd) ExistEnvFilePrint(ssh, wd)
WriteEnvFile(ssh, services, wd, self.deploymentTag, self.flexricTag) WriteEnvFile(ssh, services, wd, self.deploymentTag, self.flexricTag)
if num_attempts <= 0:
logging.info(f"will start services {services}") raise ValueError(f'Invalid value for num_attempts: {num_attempts}, must be greater than 0')
status = ssh.run(f'docker compose -f {wd}/docker-compose.y*ml up -d -- {services}') for attempt in range(num_attempts):
if status.returncode != 0: imagesInfo = []
msg = f"cannot deploy services {services}: {status.stdout}" healthInfo = []
logging.error(msg) logging.info(f'will start services {services}')
HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg]) status = ssh.run(f'docker compose -f {wd_yaml} up -d -- {services}')
return False if status.returncode != 0:
msg = f'cannot deploy services {services}: {status.stdout}'
imagesInfo = [] logging.error(msg)
fstatus = True HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [msg])
for svc in services.split(): return False
containerName = GetContainerName(ssh, svc, f"{wd}/docker-compose.y*ml") for svc in services.split():
healthy = GetContainerHealth(ssh, containerName) health, msg = GetServiceHealth(ssh, svc, f'{wd_yaml}')
if not healthy: logging.info(msg)
imagesInfo += [f"Failed to deploy: service {svc}"] imagesInfo.append(msg)
fstatus = False healthInfo.append(health)
else: deployed = all(healthInfo)
image = GetImageName(ssh, svc, f"{wd}/docker-compose.y*ml") if deployed:
logging.info(f"service {svc} healthy, container {containerName}, image {image}") break
imagesInfo += [f"service {svc} healthy, image {image}"] elif (attempt < num_attempts - 1):
if fstatus: logging.warning(f'Failed to deploy on attempt {attempt}, restart services {services}')
for svc in services.split():
CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log')
ssh.run(f'docker compose -f {wd_yaml} down -- {services}')
if deployed:
HTML.CreateHtmlTestRowQueue('N/A', 'OK', ['\n'.join(imagesInfo)]) HTML.CreateHtmlTestRowQueue('N/A', 'OK', ['\n'.join(imagesInfo)])
else: else:
HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)]) HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)])
return fstatus return deployed
def UndeployObject(self, HTML, RAN): def UndeployObject(self, HTML, RAN):
svr = self.eNB_serverId[self.eNB_instance] svr = self.eNB_serverId[self.eNB_instance]
...@@ -902,7 +905,7 @@ class Containerize(): ...@@ -902,7 +905,7 @@ class Containerize():
if services is not None: if services is not None:
all_serv = " ".join([s for s, _ in services]) all_serv = " ".join([s for s, _ in services])
ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}') ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}')
copyin_res = all(CopyinContainerLog(ssh, lSourcePath, yaml_dir, c, f'{s}-{HTML.testCase_id}.log') for s, c in services) copyin_res = all(CopyinServiceLog(ssh, lSourcePath, yaml_dir, s, f"{wd}/docker-compose.y*ml", f'{s}-{HTML.testCase_id}.log') for s, c in services)
else: else:
logging.warning('could not identify services to stop => no log file') logging.warning('could not identify services to stop => no log file')
ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v') ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v')
......
...@@ -349,6 +349,7 @@ def ExecuteActionWithParam(action): ...@@ -349,6 +349,7 @@ def ExecuteActionWithParam(action):
string_field = test.findtext('services') string_field = test.findtext('services')
if string_field is not None: if string_field is not None:
CONTAINERS.services[CONTAINERS.eNB_instance] = string_field CONTAINERS.services[CONTAINERS.eNB_instance] = string_field
CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1)
CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge) CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge)
if action == 'Deploy_Object': if action == 'Deploy_Object':
success = CONTAINERS.DeployObject(HTML) success = CONTAINERS.DeployObject(HTML)
......
...@@ -47,6 +47,7 @@ class TestDeploymentMethods(unittest.TestCase): ...@@ -47,6 +47,7 @@ class TestDeploymentMethods(unittest.TestCase):
self.cont.eNBUserName = None self.cont.eNBUserName = None
self.cont.eNBPassword = None self.cont.eNBPassword = None
self.cont.eNBSourceCodePath = os.getcwd() self.cont.eNBSourceCodePath = os.getcwd()
self.cont.num_attempts = 3
def test_deploy(self): def test_deploy(self):
self.cont.yamlPath[0] = 'tests/simple-dep/' self.cont.yamlPath[0] = 'tests/simple-dep/'
...@@ -65,6 +66,15 @@ class TestDeploymentMethods(unittest.TestCase): ...@@ -65,6 +66,15 @@ class TestDeploymentMethods(unittest.TestCase):
self.assertFalse(deploy) self.assertFalse(deploy)
self.cont.yamlPath = old self.cont.yamlPath = old
def test_deployfails_2svc(self):
# fails reliably
old = self.cont.yamlPath
self.cont.yamlPath[0] = 'tests/simple-fail-2svc/'
deploy = self.cont.DeployObject(self.html)
self.cont.UndeployObject(self.html, self.ran)
self.assertFalse(deploy)
self.cont.yamlPath = old
def test_deploy_ran(self): def test_deploy_ran(self):
self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
self.cont.services[0] = "oai-gnb" self.cont.services[0] = "oai-gnb"
......
services:
test1:
image: ubuntu:jammy
container_name: test_container1
cap_drop:
- ALL
entrypoint: /bin/bash -c "false"
healthcheck:
test: /bin/bash -c "true"
interval: 1s
timeout: 1s
retries: 1
test2:
image: ubuntu:jammy
container_name: test_container2
cap_drop:
- ALL
entrypoint: /bin/bash -c "sleep infinity"
healthcheck:
test: /bin/bash -c "true"
interval: 1s
timeout: 1s
retries: 1
...@@ -89,6 +89,7 @@ ...@@ -89,6 +89,7 @@
<yaml_path>ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz</yaml_path> <yaml_path>ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz</yaml_path>
<eNB_instance>0</eNB_instance> <eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId> <eNB_serverId>0</eNB_serverId>
<num_attempts>3</num_attempts>
</testCase> </testCase>
<testCase id="000001"> <testCase id="000001">
......
...@@ -89,6 +89,7 @@ ...@@ -89,6 +89,7 @@
<yaml_path>ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz</yaml_path> <yaml_path>ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz</yaml_path>
<eNB_instance>0</eNB_instance> <eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId> <eNB_serverId>0</eNB_serverId>
<num_attempts>3</num_attempts>
</testCase> </testCase>
<testCase id="000001"> <testCase id="000001">
......
...@@ -89,6 +89,7 @@ ...@@ -89,6 +89,7 @@
<yaml_path>ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz</yaml_path> <yaml_path>ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz</yaml_path>
<eNB_instance>0</eNB_instance> <eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId> <eNB_serverId>0</eNB_serverId>
<num_attempts>3</num_attempts>
</testCase> </testCase>
<testCase id="000001"> <testCase id="000001">
......
...@@ -80,6 +80,7 @@ ...@@ -80,6 +80,7 @@
<yaml_path>ci-scripts/yaml_files/5g_sa_n310_gnb</yaml_path> <yaml_path>ci-scripts/yaml_files/5g_sa_n310_gnb</yaml_path>
<eNB_instance>0</eNB_instance> <eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId> <eNB_serverId>0</eNB_serverId>
<num_attempts>3</num_attempts>
</testCase> </testCase>
<testCase id="800814"> <testCase id="800814">
<class>Create_Workspace</class> <class>Create_Workspace</class>
...@@ -93,6 +94,7 @@ ...@@ -93,6 +94,7 @@
<yaml_path>ci-scripts/yaml_files/5g_sa_n310_nrue</yaml_path> <yaml_path>ci-scripts/yaml_files/5g_sa_n310_nrue</yaml_path>
<eNB_instance>1</eNB_instance> <eNB_instance>1</eNB_instance>
<eNB_serverId>1</eNB_serverId> <eNB_serverId>1</eNB_serverId>
<num_attempts>3</num_attempts>
</testCase> </testCase>
<testCase id="000001"> <testCase id="000001">
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment