From 7361c1689ecafd7eb86832efe442a7afdbb97e8e Mon Sep 17 00:00:00 2001 From: Robert Schmidt <robert.schmidt@openairinterface.org> Date: Thu, 15 Dec 2022 09:27:43 +0100 Subject: [PATCH] Use new SSH class to execute image build --- ci-scripts/cls_cluster.py | 2 +- ci-scripts/cls_containerize.py | 100 ++++++++++++++++----------------- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 80f7326338..892d9e847e 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -352,7 +352,7 @@ class Cluster: self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done', '\$', 60) build_log_name = f'build_log_{self.testCase_id}' - cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name, lIpAddr, 'oaicicd', CONST.CI_NO_PASSWORD) + cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name) self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done') diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 4540f2a8a0..d766f02b8b 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -44,6 +44,7 @@ from zipfile import ZipFile #----------------------------------------------------------- # OAI Testing modules #----------------------------------------------------------- +import cls_cmd import sshconnection as SSH import helpreadme as HELP import constants as CONST @@ -83,20 +84,20 @@ def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTarge logging.debug(f'Merging with the target branch: {ranTargetBranch}') sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 30) -def CopyLogsToExecutor(sshSession, sourcePath, log_name, scpIp, scpUser, scpPw): - sshSession.command(f'cd {sourcePath}/cmake_targets', '\$', 5) - sshSession.command(f'rm -f {log_name}.zip', '\$', 5) - sshSession.command(f'mkdir -p {log_name}', '\$', 5) - sshSession.command(f'mv log/* {log_name}', '\$', 5) - sshSession.command(f'zip -r -qq {log_name}.zip {log_name}', '\$', 5) +def CopyLogsToExecutor(cmd, sourcePath, log_name): + cmd.cd(f'{sourcePath}/cmake_targets') + cmd.run(f'rm -f {log_name}.zip') + cmd.run(f'mkdir -p {log_name}') + cmd.run(f'mv log/* {log_name}') + cmd.run(f'zip -r -qq {log_name}.zip {log_name}') # copy zip to executor for analysis if (os.path.isfile(f'./{log_name}.zip')): os.remove(f'./{log_name}.zip') if (os.path.isdir(f'./{log_name}')): shutil.rmtree(f'./{log_name}') - sshSession.copyin(scpIp, scpUser, scpPw, f'{sourcePath}/cmake_targets/{log_name}.zip', '.') - sshSession.command(f'rm -f {log_name}.zip','\$', 5) + cmd.copyin(f'{sourcePath}/cmake_targets/{log_name}.zip', f'./{log_name}.zip') + cmd.run(f'rm -f {log_name}.zip') ZipFile(f'{log_name}.zip').extractall('.') def AnalyzeBuildLogs(buildRoot, images, globalStatus): @@ -329,12 +330,11 @@ class Containerize(): HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') logging.debug('Building on server: ' + lIpAddr) - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) + cmd = cls_cmd.RemoteCmd(lIpAddr) # Checking the hostname to get adapted on cli and dockerfileprefixes - mySSH.command('hostnamectl', '\$', 5) - result = re.search('Ubuntu|Red Hat', mySSH.getBefore()) + cmd.run('hostnamectl') + result = re.search('Ubuntu|Red Hat', cmd.getBefore()) self.host = result.group(0) if self.host == 'Ubuntu': self.cli = 'docker' @@ -369,18 +369,18 @@ class Containerize(): # Workaround for some servers, we need to erase completely the workspace if self.forcedWorkspaceCleanup: - mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15) + cmd.run(f'sudo -S rm -Rf {lSourcePath}') self.testCase_id = HTML.testCase_id - CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge) + CreateWorkspace(cmd, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge) # if asterix, copy the entitlement and subscription manager configurations if self.host == 'Red Hat': - mySSH.command('mkdir -p ./etc-pki-entitlement ./rhsm-conf ./rhsm-ca', '\$', 5) - mySSH.command('sudo cp /etc/rhsm/rhsm.conf ./rhsm-conf/', '\$', 5) - mySSH.command('sudo cp /etc/rhsm/ca/redhat-uep.pem ./rhsm-ca/', '\$', 5) - mySSH.command('sudo cp /etc/pki/entitlement/*.pem ./etc-pki-entitlement/', '\$', 5) + cmd.run('mkdir -p ./etc-pki-entitlement ./rhsm-conf ./rhsm-ca') + cmd.run('cp /etc/rhsm/rhsm.conf ./rhsm-conf/') + cmd.run('cp /etc/rhsm/ca/redhat-uep.pem ./rhsm-ca/') + cmd.run('cp /etc/pki/entitlement/*.pem ./etc-pki-entitlement/') baseImage = 'ran-base' baseTag = 'develop' @@ -389,8 +389,8 @@ class Containerize(): if (self.ranAllowMerge): imageTag = 'ci-temp' if self.ranTargetBranch == 'develop': - mySSH.command('git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5) - result = re.search('index', mySSH.getBefore()) + cmd.run(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base{self.dockerfileprefix} | grep --colour=never -i INDEX') + result = re.search('index', cmd.getBefore()) if result is not None: forceBaseImageBuild = True baseTag = 'ci-temp' @@ -398,31 +398,31 @@ class Containerize(): forceBaseImageBuild = True # Let's remove any previous run artifacts if still there - mySSH.command(self.cli + ' image prune --force', '\$', 30) + cmd.run(f"{self.cli} image prune --force") if forceBaseImageBuild: - mySSH.command(self.cli + ' image rm ' + baseImage + ':' + baseTag + ' || true', '\$', 30) + cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}") for image,pattern in imageNames: - mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag + ' || true', '\$', 30) + cmd.run(f"{self.cli} image rm {image}:{imageTag}") # Build the base image only on Push Events (not on Merge Requests) # On when the base image docker file is being modified. if forceBaseImageBuild: - mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + baseImage + ' --tag ' + baseImage + ':' + baseTag + ' --file docker/Dockerfile.base' + self.dockerfileprefix + ' . > cmake_targets/log/ran-base.log 2>&1', '\$', 1600) + cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> cmake_targets/log/ran-base.log", timeout=1600) # First verify if the base image was properly created. - mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + baseImage + ':' + baseTag, '\$', 5) + ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}") allImagesSize = {} - if mySSH.getBefore().count('o such image') != 0: + if ret.returncode != 0: logging.error('\u001B[1m Could not build properly ran-base\u001B[0m') # Recover the name of the failed container? - mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) - mySSH.command(self.cli + ' image prune --force', '\$', 30) - mySSH.close() + cmd.run(f"{self.cli} ps --quiet --filter \"status=exited\" -n1 | xargs {self.cli} rm -f") + cmd.run(f"{self.cli} image prune --force") + cmd.close() logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTabFooter(False) sys.exit(1) else: - result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore()) + result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore()) if result is not None: size = float(result.group("size")) / 1000000 imageSizeStr = f'{size:.1f}' @@ -432,10 +432,10 @@ class Containerize(): logging.debug('ran-base size is unknown') # Recover build logs, for the moment only possible when build is successful - mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5) - mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5) - mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5) - mySSH.command(self.cli + ' rm -f test', '\$', 5) + cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}") + cmd.run("mkdir -p cmake_targets/log/ran-base") + cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base") + cmd.run(f"{self.cli} rm -f test") # Build the target image(s) status = True @@ -444,24 +444,24 @@ class Containerize(): attemptedImages += [image] # the archived Dockerfiles have "ran-base:latest" as base image # we need to update them with proper tag - mySSH.command(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5) + cmd.run(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}') if image != 'ran-build': - mySSH.command(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5) - mySSH.command(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', '\$', 1200) + cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}') + cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', timeout=1200) # split the log - mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) - mySSH.command('python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/' + image + '.log', '\$', 5) - # checking the status of the build - mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5) - if mySSH.getBefore().count('o such image') != 0: + cmd.run(f"mkdir -p cmake_targets/log/{image}") + cmd.run(f"python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log") + # check the status of the build + ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {image}:{imageTag}") + if ret.returncode != 0: logging.error('\u001B[1m Could not build properly ' + image + '\u001B[0m') status = False # Here we should check if the last container corresponds to a failed command and destroy it - mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5) + cmd.run(f"{self.cli} ps --quiet --filter \"status=exited\" -n1 | xargs {self.cli} rm -f") allImagesSize[image] = 'N/A -- Build Failed' break else: - result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore()) + result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore()) if result is not None: size = float(result.group("size")) / 1000000 imageSizeStr = f'{size:.1f}' @@ -471,18 +471,18 @@ class Containerize(): logging.debug(f'{image} size is unknown') allImagesSize[image] = 'unknown' # Now pruning dangling images in between target builds - mySSH.command(self.cli + ' image prune --force', '\$', 30) + cmd.run(f"{self.cli} image prune --force") # Remove all intermediate build images and clean up if self.ranAllowMerge and forceBaseImageBuild: - mySSH.command(f'{self.cli} image rm {baseImage}:{baseTag} || true', '\$', 30) - mySSH.command(f'{self.cli} image rm ran-build:{imageTag}','\$', 30) - mySSH.command(f'{self.cli} volume prune --force','\$', 15) + cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}") + cmd.run(f"{self.cli} image rm ran-build:{imageTag}") + cmd.run(f"{self.cli} volume prune --force") # create a zip with all logs build_log_name = f'build_log_{self.testCase_id}' - CopyLogsToExecutor(mySSH, lSourcePath, build_log_name, lIpAddr, lUserName, lPassWord) - mySSH.close() + CopyLogsToExecutor(cmd, lSourcePath, build_log_name) + cmd.close() # Analyze the logs collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status) -- 2.26.2