Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
O
OpenXG-RAN
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Operations
Operations
Metrics
Environments
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
OpenXG
OpenXG-RAN
Commits
ab1564aa
Commit
ab1564aa
authored
9 months ago
by
Bipin Adhikari
Committed by
Raphael Defosseux
8 months ago
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
All log files are now stored in scenario-specific folders in python executor
parent
f9955241
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
26 additions
and
33 deletions
+26
-33
ci-scripts/cls_containerize.py
ci-scripts/cls_containerize.py
+16
-27
ci-scripts/cls_oaicitest.py
ci-scripts/cls_oaicitest.py
+9
-5
ci-scripts/main.py
ci-scripts/main.py
+1
-1
No files found.
ci-scripts/cls_containerize.py
View file @
ab1564aa
...
@@ -944,11 +944,10 @@ class Containerize():
...
@@ -944,11 +944,10 @@ class Containerize():
if
services_list
==
[]:
services_list
=
allServices
#handing empty services
if
services_list
==
[]:
services_list
=
allServices
#handing empty services
for
svcName
in
services_list
:
for
svcName
in
services_list
:
logging
.
debug
(
'current service in the service list:
\033
[4m%s
\033
[0m'
,
svcName
)
logging
.
debug
(
'current service in the service list:
\033
[4m%s
\033
[0m'
,
svcName
)
mySSH
.
run
(
f
'docker compose --file ci-docker-compose.yml config
{
svcName
}
'
)
mySSH
.
run
(
f
"docker compose -f docker-compose.yaml config --format json
{
svcName
}
| jq -r '.services.
\"
{
svcName
}
\"
.container_name'"
)
result
=
re
.
search
(
'container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)'
,
mySSH
.
getBefore
()
)
result
=
mySSH
.
getBefore
(
)
unhealthyNb
=
0
unhealthyNb
=
0
healthyNb
=
0
healthyNb
=
0
startingNb
=
0
containerName
=
''
containerName
=
''
usedImage
=
''
usedImage
=
''
imageInfo
=
''
imageInfo
=
''
...
@@ -961,7 +960,6 @@ class Containerize():
...
@@ -961,7 +960,6 @@ class Containerize():
mySSH
.
run
(
'docker inspect --format="{{.State.Health.Status}}" '
+
containerName
,
5
)
mySSH
.
run
(
'docker inspect --format="{{.State.Health.Status}}" '
+
containerName
,
5
)
unhealthyNb
=
mySSH
.
getBefore
().
count
(
'unhealthy'
)
unhealthyNb
=
mySSH
.
getBefore
().
count
(
'unhealthy'
)
healthyNb
=
mySSH
.
getBefore
().
count
(
'healthy'
)
-
unhealthyNb
healthyNb
=
mySSH
.
getBefore
().
count
(
'healthy'
)
-
unhealthyNb
startingNb
=
mySSH
.
getBefore
().
count
(
'starting'
)
if
healthyNb
==
1
:
if
healthyNb
==
1
:
cnt
=
10
cnt
=
10
else
:
else
:
...
@@ -971,7 +969,6 @@ class Containerize():
...
@@ -971,7 +969,6 @@ class Containerize():
# faking health of db_init
# faking health of db_init
unhealthyNb
=
0
unhealthyNb
=
0
healthyNb
=
1
healthyNb
=
1
startingNb
=
0
break
break
mySSH
.
run
(
'docker inspect --format="ImageUsed: {{.Config.Image}}" '
+
containerName
)
mySSH
.
run
(
'docker inspect --format="ImageUsed: {{.Config.Image}}" '
+
containerName
)
for
stdoutLine
in
mySSH
.
getBefore
().
split
(
'
\n
'
):
for
stdoutLine
in
mySSH
.
getBefore
().
split
(
'
\n
'
):
...
@@ -979,17 +976,9 @@ class Containerize():
...
@@ -979,17 +976,9 @@ class Containerize():
usedImage
=
stdoutLine
.
replace
(
'ImageUsed: oai-ci'
,
'oai-ci'
).
strip
()
usedImage
=
stdoutLine
.
replace
(
'ImageUsed: oai-ci'
,
'oai-ci'
).
strip
()
logging
.
debug
(
'Used image is '
+
usedImage
)
logging
.
debug
(
'Used image is '
+
usedImage
)
if
usedImage
!=
''
:
if
usedImage
!=
''
:
mySSH
.
run
(
'docker image inspect --format "* Size = {{.Size}} bytes
\n
* Creation = {{.Created}}
\n
* Id = {{.Id}}" '
+
usedImage
,
5
,
silent
=
True
)
ret
=
mySSH
.
run
(
'docker image inspect --format "* Size = {{.Size}} bytes
\n
* Creation = {{.Created}}
\n
* Id = {{.Id}}" '
+
usedImage
)
for
stdoutLine
in
mySSH
.
getBefore
().
split
(
'
\n
'
):
imageInfo
=
ret
.
stdout
if
re
.
search
(
'Size = [0-9]'
,
stdoutLine
)
is
not
None
:
logging
.
debug
(
f' -- for service
{
svcName
}
containers:
{
str
(
healthyNb
)
}
healthy,
{
str
(
healthyNb
)
}
unhealthy'
)
imageInfo
+=
stdoutLine
.
strip
()
+
'
\n
'
if
re
.
search
(
'Creation = [0-9]'
,
stdoutLine
)
is
not
None
:
imageInfo
+=
stdoutLine
.
strip
()
+
'
\n
'
if
re
.
search
(
'Id = sha256'
,
stdoutLine
)
is
not
None
:
imageInfo
+=
stdoutLine
.
strip
()
+
'
\n
'
logging
.
debug
(
' -- '
+
str
(
healthyNb
)
+
' healthy container(s) for service'
+
svcName
)
logging
.
debug
(
' -- '
+
str
(
unhealthyNb
)
+
' unhealthy container(s) for service'
+
svcName
)
logging
.
debug
(
' -- '
+
str
(
startingNb
)
+
' still starting container(s) for service'
+
svcName
)
self
.
testCase_id
=
HTML
.
testCase_id
self
.
testCase_id
=
HTML
.
testCase_id
self
.
eNB_logFile
[
self
.
eNB_instance
]
=
f'
{
svcName
}
-
{
self
.
testCase_id
}
.log'
self
.
eNB_logFile
[
self
.
eNB_instance
]
=
f'
{
svcName
}
-
{
self
.
testCase_id
}
.log'
...
@@ -1004,7 +993,7 @@ class Containerize():
...
@@ -1004,7 +993,7 @@ class Containerize():
logfilename
=
f'
{
lSourcePath
}
/cmake_targets/log/
{
self
.
eNB_logFile
[
self
.
eNB_instance
]
}
'
logfilename
=
f'
{
lSourcePath
}
/cmake_targets/log/
{
self
.
eNB_logFile
[
self
.
eNB_instance
]
}
'
mySSH
.
run
(
f'docker logs
{
containerName
}
>
{
logfilename
}
'
,
30
)
mySSH
.
run
(
f'docker logs
{
containerName
}
>
{
logfilename
}
'
,
30
)
cwd
=
os
.
getcwd
()
cwd
=
os
.
getcwd
()
logging
.
debu
g
(
f'Deployment Failed, trying to copy the log from
{
logfilename
}
at
{
cwd
}
'
)
logging
.
warnin
g
(
f'Deployment Failed, trying to copy the log from
{
logfilename
}
at
{
cwd
}
'
)
mySSH
.
copyin
(
logfilename
,
cwd
,
True
)
mySSH
.
copyin
(
logfilename
,
cwd
,
True
)
mySSH
.
close
()
mySSH
.
close
()
...
@@ -1074,11 +1063,15 @@ class Containerize():
...
@@ -1074,11 +1063,15 @@ class Containerize():
mySSH
.
run
(
f'docker compose -f
{
yamlDir
}
/ci-docker-compose.yml stop -t3'
)
mySSH
.
run
(
f'docker compose -f
{
yamlDir
}
/ci-docker-compose.yml stop -t3'
)
cwd
=
os
.
getcwd
()
cwd
=
os
.
getcwd
()
copyin_res
=
True
copyin_res
=
True
ymlPath
=
self
.
yamlPath
[
0
].
split
(
'/'
)
logPath
=
f'
{
cwd
}
/../cmake_targets/log/
{
ymlPath
[
2
]
}
'
# Creating destination log folder if needed on the python executor workspace
os
.
system
(
f'mkdir -p
{
logPath
}
'
)
for
service_name
,
container_id
in
services
:
for
service_name
,
container_id
in
services
:
# head -n -1 suppresses the final "X exited with status code Y"
# head -n -1 suppresses the final "X exited with status code Y"
filename
=
f'
{
service_name
}
-
{
HTML
.
testCase_id
}
.log'
filename
=
f'
{
service_name
}
-
{
HTML
.
testCase_id
}
.log'
mySSH
.
run
(
f'docker logs
{
container_id
}
>
{
lSourcePath
}
/cmake_targets/log/
{
filename
}
2>&1'
)
mySSH
.
run
(
f'docker logs
{
container_id
}
>
{
lSourcePath
}
/cmake_targets/log/
{
filename
}
2>&1'
)
copyin_res
=
mySSH
.
copyin
(
f'
{
lSourcePath
}
/cmake_targets/log/
{
filename
}
'
,
os
.
path
.
join
(
cwd
,
filename
))
and
copyin_res
copyin_res
=
mySSH
.
copyin
(
f'
{
lSourcePath
}
/cmake_targets/log/
{
filename
}
'
,
os
.
path
.
join
(
logPath
,
filename
))
and
copyin_res
mySSH
.
run
(
f'docker compose -f
{
yamlDir
}
/ci-docker-compose.yml down -v'
)
mySSH
.
run
(
f'docker compose -f
{
yamlDir
}
/ci-docker-compose.yml down -v'
)
...
@@ -1087,20 +1080,19 @@ class Containerize():
...
@@ -1087,20 +1080,19 @@ class Containerize():
HTML
.
htmleNBFailureMsg
=
'Could not copy logfile(s) to analyze it!'
HTML
.
htmleNBFailureMsg
=
'Could not copy logfile(s) to analyze it!'
HTML
.
CreateHtmlTestRow
(
'N/A'
,
'KO'
,
CONST
.
ENB_PROCESS_NOLOGFILE_TO_ANALYZE
)
HTML
.
CreateHtmlTestRow
(
'N/A'
,
'KO'
,
CONST
.
ENB_PROCESS_NOLOGFILE_TO_ANALYZE
)
self
.
exitStatus
=
1
self
.
exitStatus
=
1
# use function for UE log analysis, when oai-nr-ue container is used
else
:
else
:
for
service_name
,
_
in
services
:
for
service_name
,
_
in
services
:
self
.
exitStatus
==
0
self
.
exitStatus
==
0
filename
=
f'
{
service_name
}
-
{
HTML
.
testCase_id
}
.log'
filename
=
f'
{
logPath
}
/
{
service_name
}
-
{
HTML
.
testCase_id
}
.log'
if
(
any
(
sub
in
service_name
for
sub
in
[
'oai_ue'
,
'oai-nr-ue'
,
'lte_ue'
])):
if
(
any
(
sub
in
service_name
for
sub
in
[
'oai_ue'
,
'oai-nr-ue'
,
'lte_ue'
])):
logging
.
debug
(
f'
Analyzing UE logfile
{
filename
}
'
)
logging
.
debug
(
f'
\u001B
[1m Analyzing UE logfile
{
filename
}
\u001B
[0m
'
)
logStatus
=
cls_oaicitest
.
OaiCiTest
().
AnalyzeLogFile_UE
(
filename
,
HTML
,
RAN
)
logStatus
=
cls_oaicitest
.
OaiCiTest
().
AnalyzeLogFile_UE
(
filename
,
HTML
,
RAN
)
if
(
logStatus
<
0
):
if
(
logStatus
<
0
):
HTML
.
CreateHtmlTestRow
(
'UE log Analysis'
,
'KO'
,
logStatus
)
HTML
.
CreateHtmlTestRow
(
'UE log Analysis'
,
'KO'
,
logStatus
)
self
.
exitStatus
=
1
self
.
exitStatus
=
1
else
:
else
:
HTML
.
CreateHtmlTestRow
(
'UE log Analysis'
,
'OK'
,
CONST
.
ALL_PROCESSES_OK
)
HTML
.
CreateHtmlTestRow
(
'UE log Analysis'
,
'OK'
,
CONST
.
ALL_PROCESSES_OK
)
elif
(
any
(
sub
in
service_name
for
sub
in
[
'
oai_enb'
,
'oai-cu'
,
'oai-du'
,
'oai-
gnb'
])):
elif
(
any
(
sub
in
service_name
for
sub
in
[
'
enb'
,
'rru'
,
'rcc'
,
'cu'
,
'du'
,
'
gnb'
])):
logging
.
debug
(
f'
\u001B
[1m Analyzing XnB logfile
{
filename
}
\u001B
[0m'
)
logging
.
debug
(
f'
\u001B
[1m Analyzing XnB logfile
{
filename
}
\u001B
[0m'
)
logStatus
=
RAN
.
AnalyzeLogFile_eNB
(
filename
,
HTML
,
self
.
ran_checkers
)
logStatus
=
RAN
.
AnalyzeLogFile_eNB
(
filename
,
HTML
,
self
.
ran_checkers
)
if
(
logStatus
<
0
):
if
(
logStatus
<
0
):
...
@@ -1112,10 +1104,7 @@ class Containerize():
...
@@ -1112,10 +1104,7 @@ class Containerize():
msg
=
'Undeploy PNF/Nvidia CUBB'
msg
=
'Undeploy PNF/Nvidia CUBB'
HTML
.
CreateHtmlTestRow
(
msg
,
'OK'
,
CONST
.
ALL_PROCESSES_OK
)
HTML
.
CreateHtmlTestRow
(
msg
,
'OK'
,
CONST
.
ALL_PROCESSES_OK
)
else
:
else
:
logging
.
info
(
f'Skipping to analysize log for service name
{
service_name
}
'
)
logging
.
info
(
f'Skipping to analyze log for service name
{
service_name
}
'
)
# all the xNB run logs shall be on the server 0 for logCollecting
if
self
.
eNB_serverId
[
self
.
eNB_instance
]
!=
'0'
:
mySSH
.
copyout
(
f"
{
cwd
}
/*.log"
,
f'
{
lSourcePath
}
/cmake_targets/'
,
recursive
=
True
)
if
self
.
exitStatus
==
0
:
if
self
.
exitStatus
==
0
:
logging
.
info
(
'
\u001B
[1m Undeploying OAI Object Pass
\u001B
[0m'
)
logging
.
info
(
'
\u001B
[1m Undeploying OAI Object Pass
\u001B
[0m'
)
else
:
else
:
...
@@ -1356,7 +1345,7 @@ class Containerize():
...
@@ -1356,7 +1345,7 @@ class Containerize():
filenames
=
str
(
lsStatus
.
stdout
).
strip
()
filenames
=
str
(
lsStatus
.
stdout
).
strip
()
for
filename
in
filenames
.
split
(
'
\n
'
):
for
filename
in
filenames
.
split
(
'
\n
'
):
logging
.
debug
(
'
\u001B
[1m Analyzing UE logfile '
+
filename
+
'
\u001B
[0m'
)
logging
.
debug
(
f'
\u001B
[1m Analyzing UE logfile
{
filename
}
\u001B
[0m'
)
logStatus
=
UE
.
AnalyzeLogFile_UE
(
f'
{
logPath
}
/
{
filename
}
'
,
HTML
,
RAN
)
logStatus
=
UE
.
AnalyzeLogFile_UE
(
f'
{
logPath
}
/
{
filename
}
'
,
HTML
,
RAN
)
if
(
logStatus
<
0
):
if
(
logStatus
<
0
):
fullStatus
=
False
fullStatus
=
False
...
...
This diff is collapsed.
Click to expand it.
ci-scripts/cls_oaicitest.py
View file @
ab1564aa
...
@@ -670,7 +670,7 @@ class OaiCiTest():
...
@@ -670,7 +670,7 @@ class OaiCiTest():
messages
=
[
f
.
result
()
for
f
in
futures
]
messages
=
[
f
.
result
()
for
f
in
futures
]
HTML
.
CreateHtmlTestRowQueue
(
'NA'
,
'OK'
,
messages
)
HTML
.
CreateHtmlTestRowQueue
(
'NA'
,
'OK'
,
messages
)
def
Ping_common
(
self
,
EPC
,
ue
,
RAN
,
printLock
):
def
Ping_common
(
self
,
EPC
,
ue
,
RAN
,
CONTAINERS
,
printLock
):
# Launch ping on the EPC side (true for ltebox and old open-air-cn)
# Launch ping on the EPC side (true for ltebox and old open-air-cn)
ping_status
=
0
ping_status
=
0
ueIP
=
ue
.
getIP
()
ueIP
=
ue
.
getIP
()
...
@@ -678,7 +678,11 @@ class OaiCiTest():
...
@@ -678,7 +678,11 @@ class OaiCiTest():
return
(
False
,
f"UE
{
ue
.
getName
()
}
has no IP address"
)
return
(
False
,
f"UE
{
ue
.
getName
()
}
has no IP address"
)
ping_log_file
=
f'ping_
{
self
.
testCase_id
}
_
{
ue
.
getName
()
}
.log'
ping_log_file
=
f'ping_
{
self
.
testCase_id
}
_
{
ue
.
getName
()
}
.log'
ping_time
=
re
.
findall
(
"-c *(\d+)"
,
str
(
self
.
ping_args
))
ping_time
=
re
.
findall
(
"-c *(\d+)"
,
str
(
self
.
ping_args
))
local_ping_log_file
=
f'
{
os
.
getcwd
()
}
/
{
ping_log_file
}
'
# Creating destination log folder if needed on the python executor workspace
ymlPath
=
CONTAINERS
.
yamlPath
[
0
].
split
(
'/'
)
logPath
=
f'../cmake_targets/log/
{
ymlPath
[
2
]
}
'
os
.
system
(
f'mkdir -p
{
logPath
}
'
)
local_ping_log_file
=
f'
{
logPath
}
/
{
ping_log_file
}
'
# if has pattern %cn_ip%, replace with core IP address, else we assume the IP is present
# if has pattern %cn_ip%, replace with core IP address, else we assume the IP is present
if
re
.
search
(
'%cn_ip%'
,
self
.
ping_args
):
if
re
.
search
(
'%cn_ip%'
,
self
.
ping_args
):
#target address is different depending on EPC type
#target address is different depending on EPC type
...
@@ -772,7 +776,7 @@ class OaiCiTest():
...
@@ -772,7 +776,7 @@ class OaiCiTest():
logging
.
debug
(
ues
)
logging
.
debug
(
ues
)
pingLock
=
Lock
()
pingLock
=
Lock
()
with
concurrent
.
futures
.
ThreadPoolExecutor
()
as
executor
:
with
concurrent
.
futures
.
ThreadPoolExecutor
()
as
executor
:
futures
=
[
executor
.
submit
(
self
.
Ping_common
,
EPC
,
ue
,
RAN
,
pingLock
)
for
ue
in
ues
]
futures
=
[
executor
.
submit
(
self
.
Ping_common
,
EPC
,
ue
,
RAN
,
CONTAINERS
,
pingLock
)
for
ue
in
ues
]
results
=
[
f
.
result
()
for
f
in
futures
]
results
=
[
f
.
result
()
for
f
in
futures
]
# each result in results is a tuple, first member goes to successes, second to messages
# each result in results is a tuple, first member goes to successes, second to messages
successes
,
messages
=
map
(
list
,
zip
(
*
results
))
successes
,
messages
=
map
(
list
,
zip
(
*
results
))
...
@@ -915,9 +919,9 @@ class OaiCiTest():
...
@@ -915,9 +919,9 @@ class OaiCiTest():
self
.
AutoTerminateUEandeNB
(
HTML
,
RAN
,
EPC
,
CONTAINERS
)
self
.
AutoTerminateUEandeNB
(
HTML
,
RAN
,
EPC
,
CONTAINERS
)
def
AnalyzeLogFile_UE
(
self
,
UElogFile
,
HTML
,
RAN
):
def
AnalyzeLogFile_UE
(
self
,
UElogFile
,
HTML
,
RAN
):
if
(
not
os
.
path
.
isfile
(
f'
./
{
UElogFile
}
'
)):
if
(
not
os
.
path
.
isfile
(
f'
{
UElogFile
}
'
)):
return
-
1
return
-
1
ue_log_file
=
open
(
f'
./
{
UElogFile
}
'
,
'r'
)
ue_log_file
=
open
(
f'
{
UElogFile
}
'
,
'r'
)
exitSignalReceived
=
False
exitSignalReceived
=
False
foundAssertion
=
False
foundAssertion
=
False
msgAssertion
=
''
msgAssertion
=
''
...
...
This diff is collapsed.
Click to expand it.
ci-scripts/main.py
View file @
ab1564aa
...
@@ -554,7 +554,7 @@ elif re.match('^LogCollecteNB$', mode, re.IGNORECASE):
...
@@ -554,7 +554,7 @@ elif re.match('^LogCollecteNB$', mode, re.IGNORECASE):
if
RAN
.
eNBIPAddress
==
''
or
RAN
.
eNBUserName
==
''
or
RAN
.
eNBPassword
==
''
or
RAN
.
eNBSourceCodePath
==
''
:
if
RAN
.
eNBIPAddress
==
''
or
RAN
.
eNBUserName
==
''
or
RAN
.
eNBPassword
==
''
or
RAN
.
eNBSourceCodePath
==
''
:
HELP
.
GenericHelp
(
CONST
.
Version
)
HELP
.
GenericHelp
(
CONST
.
Version
)
sys
.
exit
(
'Insufficient Parameter'
)
sys
.
exit
(
'Insufficient Parameter'
)
if
RAN
.
eNBIPAddress
==
'none'
:
if
os
.
path
.
isdir
(
'cmake_targets/log'
)
:
cmd
=
'zip -r enb.log.'
+
RAN
.
BuildId
+
'.zip cmake_targets/log'
cmd
=
'zip -r enb.log.'
+
RAN
.
BuildId
+
'.zip cmake_targets/log'
logging
.
info
(
cmd
)
logging
.
info
(
cmd
)
try
:
try
:
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment