Commit 6a36db45 authored by Robert Schmidt's avatar Robert Schmidt

Repair and refactor proxy build function

Recent refactoring replaced the CreateWorkspace function call with a
dedicated XML step (for clarity, and because it avoids needless
recreation of workspaces). In the proxy function, the same
CreateWorkspace xml step has been added, cloning the OAI RAN repo, when
we actually need the repository for the proxy. Hence, fix this by
calling the CreateWorkspace() function manually, but only when
necessary, to build the proxy. (It would only be a problem if the
pipeline would rebuild the proxy, which did not happen because not much
work is happening on the proxy repo).

Further, refactoring and simplication of the proxy build function were
done:

- never tag proxy with develop, that makes no sense
- overwrite CreateWorkspace() info after having created and used the
  workspace (the "if" to build the proxy)
- simplify copying of logs
- add simple unit test
- remove check for ubuntu; the dependcy is "docker". Simplify the code
  to not obscure with a class variable.
parent cf576fc2
...@@ -601,21 +601,7 @@ class Containerize(): ...@@ -601,21 +601,7 @@ class Containerize():
logging.debug('Building on server: ' + lIpAddr) logging.debug('Building on server: ' + lIpAddr)
ssh = cls_cmd.getConnection(lIpAddr) ssh = cls_cmd.getConnection(lIpAddr)
# Check that we are on Ubuntu self.testCase_id = HTML.testCase_id
ret = ssh.run('hostnamectl')
result = re.search('Ubuntu', ret.stdout)
self.host = result.group(0)
if self.host != 'Ubuntu':
ssh.close()
raise Exception('Can build proxy only on Ubuntu server')
self.cli = 'docker'
self.cliBuildOptions = ''
# Workaround for some servers, we need to erase completely the workspace
if self.forcedWorkspaceCleanup:
ssh.run(f'echo {lPassWord} | sudo -S rm -Rf {lSourcePath}')
oldRanCommidID = self.ranCommitID oldRanCommidID = self.ranCommitID
oldRanRepository = self.ranRepository oldRanRepository = self.ranRepository
oldRanAllowMerge = self.ranAllowMerge oldRanAllowMerge = self.ranAllowMerge
...@@ -624,26 +610,35 @@ class Containerize(): ...@@ -624,26 +610,35 @@ class Containerize():
self.ranRepository = 'https://github.com/EpiSci/oai-lte-5g-multi-ue-proxy.git' self.ranRepository = 'https://github.com/EpiSci/oai-lte-5g-multi-ue-proxy.git'
self.ranAllowMerge = False self.ranAllowMerge = False
self.ranTargetBranch = 'master' self.ranTargetBranch = 'master'
ssh.cd(lSourcePath)
# to prevent accidentally overwriting data that might be used later
self.ranCommitID = oldRanCommidID
self.ranRepository = oldRanRepository
self.ranAllowMerge = oldRanAllowMerge
self.ranTargetBranch = oldRanTargetBranch
# Let's remove any previous run artifacts if still there # Let's remove any previous run artifacts if still there
ssh.run(f'{self.cli} image prune --force') ssh.run('docker image prune --force')
# Remove any previous proxy image # Remove any previous proxy image
ssh.run(f'{self.cli} image rm oai-lte-multi-ue-proxy:latest') ssh.run('docker image rm oai-lte-multi-ue-proxy:latest')
tag = self.proxyCommit tag = self.proxyCommit
logging.debug('building L2sim proxy image for tag ' + tag) logging.debug('building L2sim proxy image for tag ' + tag)
# check if the corresponding proxy image with tag exists. If not, build it # check if the corresponding proxy image with tag exists. If not, build it
ret = ssh.run(f'{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
if ret.returncode != 0: buildProxy = ret.returncode != 0 # if no image, build new proxy
ssh.run(f'{self.cli} build {self.cliBuildOptions} --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file docker/Dockerfile.ubuntu18.04 . > cmake_targets/log/proxy-build.log 2>&1') if buildProxy:
ssh.run(f'{self.cli} image prune --force') ssh.run(f'rm -Rf {lSourcePath}')
ret = ssh.run(f'{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') success = CreateWorkspace(lIpAddr, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
if not success:
raise Exception("could not clone proxy repository")
filename = f'build_log_{self.testCase_id}'
fullpath = f'{lSourcePath}/{filename}'
ssh.run(f'docker build --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file {lSourcePath}/docker/Dockerfile.ubuntu18.04 {lSourcePath} > {fullpath} 2>&1')
ssh.run(f'zip -r -qq {fullpath}.zip {fullpath}')
local_file = f"{os.getcwd()}/../cmake_targets/log/{filename}.zip"
ssh.copyin(f'{fullpath}.zip', local_file)
# don't delete such that we might recover the zips
#ssh.run(f'rm -f {fullpath}.zip')
ssh.run('docker image prune --force')
ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
if ret.returncode != 0: if ret.returncode != 0:
logging.error('\u001B[1m Build of L2sim proxy failed\u001B[0m') logging.error('\u001B[1m Build of L2sim proxy failed\u001B[0m')
ssh.close() ssh.close()
...@@ -656,10 +651,6 @@ class Containerize(): ...@@ -656,10 +651,6 @@ class Containerize():
# retag the build images to that we pick it up later # retag the build images to that we pick it up later
ssh.run(f'docker image tag proxy:{tag} oai-lte-multi-ue-proxy:latest') ssh.run(f'docker image tag proxy:{tag} oai-lte-multi-ue-proxy:latest')
# no merge: is a push to develop, tag the image so we can push it to the registry
if not self.ranAllowMerge:
ssh.run('docker image tag proxy:{tag} proxy:develop')
# we assume that the host on which this is built will also run the proxy. The proxy # we assume that the host on which this is built will also run the proxy. The proxy
# currently requires the following command, and the docker-compose up mechanism of # currently requires the following command, and the docker-compose up mechanism of
# the CI does not allow to run arbitrary commands. Note that the following actually # the CI does not allow to run arbitrary commands. Note that the following actually
...@@ -667,21 +658,11 @@ class Containerize(): ...@@ -667,21 +658,11 @@ class Containerize():
logging.warning('the following command belongs to deployment, but no mechanism exists to exec it there!') logging.warning('the following command belongs to deployment, but no mechanism exists to exec it there!')
ssh.run('sudo ifconfig lo: 127.0.0.2 netmask 255.0.0.0 up') ssh.run('sudo ifconfig lo: 127.0.0.2 netmask 255.0.0.0 up')
# Analyzing the logs # to prevent accidentally overwriting data that might be used later
if buildProxy: self.ranCommitID = oldRanCommidID
self.testCase_id = HTML.testCase_id self.ranRepository = oldRanRepository
ssh.cd(f'{lSourcePath}/cmake_targets') self.ranAllowMerge = oldRanAllowMerge
ssh.run(f'mkdir -p proxy_build_log_{self.testCase_id}') self.ranTargetBranch = oldRanTargetBranch
ssh.run(f'mv log/* proxy_build_log_{self.testCase_id}')
if (os.path.isfile('./proxy_build_log_' + self.testCase_id + '.zip')):
os.remove('./proxy_build_log_' + self.testCase_id + '.zip')
if (os.path.isdir('./proxy_build_log_' + self.testCase_id)):
shutil.rmtree('./proxy_build_log_' + self.testCase_id)
ssh.run(f'zip -r -qq proxy_build_log_{self.testCase_id}.zip proxy_build_log_{self.testCase_id}')
filename = f'build_log_{self.testCase_id}.zip'
ssh.copyin(f'{lSourcePath}/cmake_targets/{filename}', filename)
# don't delete such that we might recover the zips
#ssh.run(f'rm -f build_log_{self.testCase_id}.zip')
# we do not analyze the logs (we assume the proxy builds fine at this stage), # we do not analyze the logs (we assume the proxy builds fine at this stage),
# but need to have the following information to correctly display the HTML # but need to have the following information to correctly display the HTML
...@@ -696,7 +677,7 @@ class Containerize(): ...@@ -696,7 +677,7 @@ class Containerize():
ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', ret.stdout) result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', ret.stdout)
# Cleaning any created tmp volume # Cleaning any created tmp volume
ssh.run(f'{self.cli} volume prune --force') ssh.run('docker volume prune --force')
ssh.close() ssh.close()
allImagesSize = {} allImagesSize = {}
......
...@@ -6,6 +6,7 @@ scripts. ...@@ -6,6 +6,7 @@ scripts.
There are some unit tests. From the parent directory, i.e., `ci-scripts/`, There are some unit tests. From the parent directory, i.e., `ci-scripts/`,
start with start with
python tests/build.py -v
python tests/cmd.py -v python tests/cmd.py -v
python tests/deployment.py -v python tests/deployment.py -v
python tests/iperf-analysis.py -v python tests/iperf-analysis.py -v
......
import sys
import logging
logging.basicConfig(
level=logging.DEBUG,
stream=sys.stdout,
format="[%(asctime)s] %(levelname)8s: %(message)s"
)
import unittest
import tempfile
sys.path.append('./') # to find OAI imports below
import cls_oai_html
import cls_containerize
import cls_cmd
class TestBuild(unittest.TestCase):
def setUp(self):
self.html = cls_oai_html.HTMLManagement()
self.html.testCase_id = "000000"
self.cont = cls_containerize.Containerize()
self.cont.eNB_serverId[0] = '0'
self.cont.eNBIPAddress = 'localhost'
self.cont.eNBUserName = None
self.cont.eNBPassword = None
self._d = tempfile.mkdtemp()
logging.warning(f"temporary directory: {self._d}")
self.cont.eNBSourceCodePath = self._d
def tearDown(self):
logging.warning(f"removing directory contents")
with cls_cmd.getConnection(None) as cmd:
cmd.run(f"rm -rf {self._d}")
def test_build_proxy(self):
self.cont.proxyCommit = "b64d9bce986b38ca59e8582864ade3fcdd05c0dc"
success = self.cont.BuildProxy(self.html)
self.assertTrue(success)
if __name__ == '__main__':
unittest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment