Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
O
OpenXG-RAN
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
zzha zzha
OpenXG-RAN
Commits
668d0865
Commit
668d0865
authored
2 years ago
by
Robert Schmidt
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Simplify cls_containerize.BuildImage() and drop allImagesSize class member
parent
d78ccd64
Branches unavailable
2024.w06
2024.w05
2024.w04
2024.w03
2024.w02
2024.w01
2023.w51
2023.w50
2023.w49
2023.w48
2023.w47
2023.w45
2023.w43
2023.w42
2023.w41
2023.w40
2023.w39
2023.w38
2023.w37
2023.w36
2023.w34
2023.w33
2023.w32
2023.w31
2023.w30
2023.w29
2023.w28
2023.w27
2023.w26
2023.w25
2023.w24
2023.w23
2023.w22
2023.w21
2023.w20
2023.w19
2023.w18
2023.w18b
2023.w16
2023.w15
2023.w14
2023.w13
2023.w12
2023.w11
2023.w11b
2023.w10
2023.w10b
2023.w09
2023.w08
2023.w08b
2023.w07
2023.w06
2023.w05
2023.w03
2023.w02
2022.42
2022.41
2022.w51
2022.w50
2022.w49
2022.w48
2022.w47
2022.w46
2022.w45
2022.w43
2022.w42
2022.w42b
2022.w41
2022.w40
2022.w39
2022.w38
2022.w37
2022.w37b
v2.1.0
v2.0.0
flexran-eol
No related merge requests found
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
36 additions
and
56 deletions
+36
-56
ci-scripts/cls_containerize.py
ci-scripts/cls_containerize.py
+36
-56
No files found.
ci-scripts/cls_containerize.py
View file @
668d0865
...
...
@@ -163,7 +163,6 @@ class Containerize():
self
.
cliBuildOptions
=
''
self
.
dockerfileprefix
=
''
self
.
host
=
''
self
.
allImagesSize
=
{}
self
.
deployedContainers
=
[]
self
.
tsharkStarted
=
False
...
...
@@ -318,32 +317,10 @@ class Containerize():
if
forceBaseImageBuild
:
mySSH
.
command
(
self
.
cli
+
' build '
+
self
.
cliBuildOptions
+
' --target '
+
baseImage
+
' --tag '
+
baseImage
+
':'
+
baseTag
+
' --file docker/Dockerfile.base'
+
self
.
dockerfileprefix
+
' . > cmake_targets/log/ran-base.log 2>&1'
,
'\$'
,
1600
)
# First verify if the base image was properly created.
status
=
True
mySSH
.
command
(
self
.
cli
+
' image inspect --format=
\'
Size = {{.Size}} bytes
\'
'
+
baseImage
+
':'
+
baseTag
,
'\$'
,
5
)
allImagesSize
=
{}
if
mySSH
.
getBefore
().
count
(
'o such image'
)
!=
0
:
logging
.
error
(
'
\u001B
[1m Could not build properly ran-base
\u001B
[0m'
)
status
=
False
else
:
result
=
re
.
search
(
'Size *= *(?P<size>[0-9\-]+) *bytes'
,
mySSH
.
getBefore
())
if
result
is
not
None
:
imageSize
=
float
(
result
.
group
(
'size'
))
imageSize
=
imageSize
/
1000
if
imageSize
<
1000
:
logging
.
debug
(
'
\u001B
[1m ran-base size is '
+
(
'%.0f'
%
imageSize
)
+
' kbytes
\u001B
[0m'
)
self
.
allImagesSize
[
'ran-base'
]
=
str
(
round
(
imageSize
,
1
))
+
' kbytes'
else
:
imageSize
=
imageSize
/
1000
if
imageSize
<
1000
:
logging
.
debug
(
'
\u001B
[1m ran-base size is '
+
(
'%.0f'
%
imageSize
)
+
' Mbytes
\u001B
[0m'
)
self
.
allImagesSize
[
'ran-base'
]
=
str
(
round
(
imageSize
,
1
))
+
' Mbytes'
else
:
imageSize
=
imageSize
/
1000
logging
.
debug
(
'
\u001B
[1m ran-base size is '
+
(
'%.3f'
%
imageSize
)
+
' Gbytes
\u001B
[0m'
)
self
.
allImagesSize
[
'ran-base'
]
=
str
(
round
(
imageSize
,
1
))
+
' Gbytes'
else
:
logging
.
debug
(
'ran-base size is unknown'
)
# If the base image failed, no need to continue
if
not
status
:
# Recover the name of the failed container?
mySSH
.
command
(
self
.
cli
+
' ps --quiet --filter "status=exited" -n1 | xargs '
+
self
.
cli
+
' rm -f'
,
'\$'
,
5
)
mySSH
.
command
(
self
.
cli
+
' image prune --force'
,
'\$'
,
30
)
...
...
@@ -353,6 +330,15 @@ class Containerize():
HTML
.
CreateHtmlTabFooter
(
False
)
sys
.
exit
(
1
)
else
:
result
=
re
.
search
(
'Size *= *(?P<size>[0-9\-]+) *bytes'
,
mySSH
.
getBefore
())
if
result
is
not
None
:
size
=
float
(
result
.
group
(
"size"
))
/
1000000
imageSizeStr
=
f'
{
size
:.
1
f
}
'
logging
.
debug
(
f'
\u001B
[1m ran-base size is
{
imageSizeStr
}
Mbytes
\u001B
[0m'
)
allImagesSize
[
'ran-base'
]
=
f'
{
imageSizeStr
}
Mbytes'
else
:
logging
.
debug
(
'ran-base size is unknown'
)
# Recover build logs, for the moment only possible when build is successful
mySSH
.
command
(
self
.
cli
+
' create --name test '
+
baseImage
+
':'
+
baseTag
,
'\$'
,
5
)
mySSH
.
command
(
'mkdir -p cmake_targets/log/ran-base'
,
'\$'
,
5
)
...
...
@@ -360,13 +346,16 @@ class Containerize():
mySSH
.
command
(
self
.
cli
+
' rm -f test'
,
'\$'
,
5
)
# Build the target image(s)
status
=
True
attemptedImages
=
[
'ran-base'
]
for
image
,
pattern
in
imageNames
:
attemptedImages
+=
[
image
]
# the archived Dockerfiles have "ran-base:latest" as base image
# we need to update them with proper tag
mySSH
.
command
(
'sed -i -e "s#'
+
baseImage
+
':latest#'
+
baseImage
+
':'
+
baseTag
+
'#" docker/Dockerfile.'
+
pattern
+
self
.
dockerfileprefix
,
'\$'
,
5
)
mySSH
.
command
(
f'sed -i -e "s#
{
baseImage
}
:latest#
{
baseImage
}
:
{
baseTag
}
#" docker/Dockerfile.
{
pattern
}{
self
.
dockerfileprefix
}
'
,
'\$'
,
5
)
if
image
!=
'ran-build'
:
mySSH
.
command
(
'sed -i -e "s#'
+
"ran-build"
+
':latest#'
+
"ran-build"
+
':'
+
imageTag
+
'#" docker/Dockerfile.'
+
pattern
+
self
.
dockerfileprefix
,
'\$'
,
5
)
mySSH
.
command
(
self
.
cli
+
' build '
+
self
.
cliBuildOptions
+
' --target '
+
image
+
' --tag '
+
image
+
':'
+
imageTag
+
' --file docker/Dockerfile.'
+
pattern
+
self
.
dockerfileprefix
+
' . > cmake_targets/log/'
+
image
+
'
.log 2>&1'
,
'\$'
,
1200
)
mySSH
.
command
(
f'sed -i -e "s#ran-build:latest#ran-build:
{
imageTag
}
#" docker/Dockerfile.
{
pattern
}{
self
.
dockerfileprefix
}
'
,
'\$'
,
5
)
mySSH
.
command
(
f'
{
self
.
cli
}
build
{
self
.
cliBuildOptions
}
--target
{
image
}
--tag
{
image
}
:
{
imageTag
}
--file docker/Dockerfile.
{
pattern
}{
self
.
dockerfileprefix
}
. > cmake_targets/log/
{
image
}
.log 2>&1'
,
'\$'
,
1200
)
# Flatten Image
if
image
!=
'ran-build'
:
mySSH
.
command
(
'python3 ./ci-scripts/flatten_image.py --tag '
+
image
+
':'
+
imageTag
,
'\$'
,
300
)
...
...
@@ -380,27 +369,18 @@ class Containerize():
status
=
False
# Here we should check if the last container corresponds to a failed command and destroy it
mySSH
.
command
(
self
.
cli
+
' ps --quiet --filter "status=exited" -n1 | xargs '
+
self
.
cli
+
' rm -f'
,
'\$'
,
5
)
self
.
allImagesSize
[
image
]
=
'N/A -- Build Failed'
allImagesSize
[
image
]
=
'N/A -- Build Failed'
break
else
:
result
=
re
.
search
(
'Size *= *(?P<size>[0-9\-]+) *bytes'
,
mySSH
.
getBefore
())
if
result
is
not
None
:
imageSize
=
float
(
result
.
group
(
'size'
))
imageSize
=
imageSize
/
1000
if
imageSize
<
1000
:
logging
.
debug
(
'
\u001B
[1m '
+
image
+
' size is '
+
(
'%.0f'
%
imageSize
)
+
' kbytes
\u001B
[0m'
)
self
.
allImagesSize
[
image
]
=
str
(
round
(
imageSize
,
1
))
+
' kbytes'
else
:
imageSize
=
imageSize
/
1000
if
imageSize
<
1000
:
logging
.
debug
(
'
\u001B
[1m '
+
image
+
' size is '
+
(
'%.0f'
%
imageSize
)
+
' Mbytes
\u001B
[0m'
)
self
.
allImagesSize
[
image
]
=
str
(
round
(
imageSize
,
1
))
+
' Mbytes'
size
=
float
(
result
.
group
(
"size"
))
/
1000000
imageSizeStr
=
f'
{
size
:.
1
f
}
'
logging
.
debug
(
f'
\u001B
[1m
{
image
}
size is
{
imageSizeStr
}
Mbytes
\u001B
[0m'
)
allImagesSize
[
image
]
=
f'
{
imageSizeStr
}
Mbytes'
else
:
imageSize
=
imageSize
/
1000
logging
.
debug
(
'
\u001B
[1m '
+
image
+
' size is '
+
(
'%.3f'
%
imageSize
)
+
' Gbytes
\u001B
[0m'
)
self
.
allImagesSize
[
image
]
=
str
(
round
(
imageSize
,
1
))
+
' Gbytes'
else
:
logging
.
debug
(
'ran-base size is unknown'
)
self
.
allImagesSize
[
image
]
=
'unknown'
logging
.
debug
(
f'
{
image
}
size is unknown'
)
allImagesSize
[
image
]
=
'unknown'
# Now pruning dangling images in between target builds
mySSH
.
command
(
self
.
cli
+
' image prune --force'
,
'\$'
,
30
)
...
...
@@ -416,17 +396,16 @@ class Containerize():
mySSH
.
close
()
# Analyze the logs
images
=
[
'ran-base'
]
+
[
tpl
[
0
]
for
tpl
in
imageNames
]
collectInfo
=
AnalyzeBuildLogs
(
build_log_name
,
images
,
status
)
collectInfo
=
AnalyzeBuildLogs
(
build_log_name
,
attemptedImages
,
status
)
if
status
:
logging
.
info
(
'
\u001B
[1m Building OAI Image(s) Pass
\u001B
[0m'
)
HTML
.
CreateHtmlTestRow
(
self
.
imageKind
,
'OK'
,
CONST
.
ALL_PROCESSES_OK
)
HTML
.
CreateHtmlNextTabHeaderTestRow
(
collectInfo
,
self
.
allImagesSize
)
HTML
.
CreateHtmlNextTabHeaderTestRow
(
collectInfo
,
allImagesSize
)
else
:
logging
.
error
(
'
\u001B
[1m Building OAI Images Failed
\u001B
[0m'
)
HTML
.
CreateHtmlTestRow
(
self
.
imageKind
,
'KO'
,
CONST
.
ALL_PROCESSES_OK
)
HTML
.
CreateHtmlNextTabHeaderTestRow
(
collectInfo
,
self
.
allImagesSize
)
HTML
.
CreateHtmlNextTabHeaderTestRow
(
collectInfo
,
allImagesSize
)
HTML
.
CreateHtmlTabFooter
(
False
)
sys
.
exit
(
1
)
...
...
@@ -552,13 +531,14 @@ class Containerize():
collectInfo
[
'proxy'
]
=
files
mySSH
.
command
(
'docker image inspect --format=
\'
Size = {{.Size}} bytes
\'
proxy:'
+
tag
,
'\$'
,
5
)
result
=
re
.
search
(
'Size *= *(?P<size>[0-9\-]+) *bytes'
,
mySSH
.
getBefore
())
allImagesSize
=
{}
if
result
is
not
None
:
imageSize
=
float
(
result
.
group
(
'size'
))
/
1000000
logging
.
debug
(
'
\u001B
[1m proxy size is '
+
(
'%.0f'
%
imageSize
)
+
' Mbytes
\u001B
[0m'
)
self
.
allImagesSize
[
'proxy'
]
=
str
(
round
(
imageSize
,
1
))
+
' Mbytes'
allImagesSize
[
'proxy'
]
=
str
(
round
(
imageSize
,
1
))
+
' Mbytes'
else
:
logging
.
debug
(
'proxy size is unknown'
)
self
.
allImagesSize
[
'proxy'
]
=
'unknown'
allImagesSize
[
'proxy'
]
=
'unknown'
# Cleaning any created tmp volume
mySSH
.
command
(
self
.
cli
+
' volume prune --force || true'
,
'\$'
,
15
)
...
...
@@ -566,7 +546,7 @@ class Containerize():
logging
.
info
(
'
\u001B
[1m Building L2sim Proxy Image Pass
\u001B
[0m'
)
HTML
.
CreateHtmlTestRow
(
'commit '
+
tag
,
'OK'
,
CONST
.
ALL_PROCESSES_OK
)
HTML
.
CreateHtmlNextTabHeaderTestRow
(
collectInfo
,
self
.
allImagesSize
)
HTML
.
CreateHtmlNextTabHeaderTestRow
(
collectInfo
,
allImagesSize
)
def
Copy_Image_to_Test_Server
(
self
,
HTML
):
imageTag
=
'develop'
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment