Commit 7dbd3a8b authored by Bipin Adhikari's avatar Bipin Adhikari

experiment(ci): exploration on the swarm deployment for 4G RF simulator

parent 1cae1879
......@@ -177,8 +177,8 @@ eNBs =
enable_measurement_reports = "no";
////////// MME parameters:
mme_ip_address = ( { ipv4 = "192.168.61.3";
ipv6 = "192:168:30::17";
mme_ip_address = ( { ipv4 = "mme_placeholder"; ## need to resolve this###################################################
ipv6 = "192:168:30::17";
port = 36412 ;
active = "yes";
preference = "ipv4";
......@@ -195,11 +195,11 @@ eNBs =
NETWORK_INTERFACES :
{
ENB_INTERFACE_NAME_FOR_S1_MME = "eth0";
ENB_IPV4_ADDRESS_FOR_S1_MME = "192.168.61.20";
ENB_IPV4_ADDRESS_FOR_S1_MME = "enb0_placeholder"; ## need to resolve this as well#####################################
ENB_INTERFACE_NAME_FOR_S1U = "eth0";
ENB_IPV4_ADDRESS_FOR_S1U = "192.168.61.20";
ENB_IPV4_ADDRESS_FOR_S1U = "enb0_placeholder"; ## need to resolve this as well#####################################
ENB_PORT_FOR_S1U = 2152; # Spec 2152
ENB_IPV4_ADDRESS_FOR_X2C = "192.168.61.20";
ENB_IPV4_ADDRESS_FOR_X2C = "enb0_placeholder";## need to resolve this as well#####################################
ENB_PORT_FOR_X2C = 36422; # Spec 36422
};
}
......
version: '3.8'
services:
cassandra:
image: cassandra:2.1
container_name: rfsim4g-cassandra
networks:
private_net:
environment:
CASSANDRA_CLUSTER_NAME: "OAI HSS Cluster"
CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch
healthcheck:
test: /bin/bash -c "nodetool status"
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
db_init:
image: cassandra:2.1
container_name: rfsim4g-db-init
deploy:
restart_policy:
condition: on-failure
max_attempts: 10
networks:
private_net:
volumes:
- ./oai_db.cql:/home/oai_db.cql
entrypoint: /bin/bash -c "cqlsh --file /home/oai_db.cql cassandra && echo 'OK' && sleep infinity"
deploy:
placement:
constraints:
- node.labels.machine == carabe
oai_hss:
image: oaisoftwarealliance/oai-hss:latest
container_name: rfsim4g-oai-hss
hostname: hss
depends_on: [cassandra]
networks:
private_net:
public_net:
environment:
REALM: openairinterface.org
HSS_FQDN: hss.openairinterface.org
PREFIX: /openair-hss/etc
cassandra_Server_IP: cassandra
OP_KEY: 1006020f0a478bf6b699f15c062e42b3
LTE_K: fec86ba6eb707ed08905757b1bb44b8f
APN1: oai.ipv4
APN2: internet
FIRST_IMSI: 208960100000001
NB_USERS: 10
healthcheck:
test: /bin/bash -c "pgrep oai_hss"
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
redis:
image: redis:6.0.5
container_name: rfsim4g-redis
hostname: redis_4g
networks:
public_net:
volumes:
- ./redis_extern.conf:/usr/local/etc/redis/redis.conf
entrypoint: /bin/bash -c "redis-server /usr/local/etc/redis/redis.conf"
healthcheck:
test: /bin/bash -c "redis-cli -h redis_4g -p 6380 ping"
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
magma_mme:
image: oaisoftwarealliance/magma-mme:latest
container_name: rfsim4g-magma-mme
hostname: mme
depends_on: [oai_hss, redis]
networks:
public_net:
environment:
TZ: Europe/Paris
REALM: openairinterface.org
PREFIX: /openair-mme/etc
HSS_HOSTNAME: hss
HSS_FQDN: hss.openairinterface.org
HSS_REALM: openairinterface.org
MME_FQDN: mme.openairinterface.org
FEATURES: mme_oai
volumes:
- ./mme_fd.sprint.conf:/magma-mme/etc/mme_fd.conf.tmplt
- ./mme.conf:/magma-mme/etc/mme.conf.tmplt
- ./entrypoint.sh:/magma-mme/bin/entrypoint.sh
entrypoint: /bin/bash -c "/magma-mme/bin/entrypoint.sh"
healthcheck:
test: /bin/bash -c "pgrep oai_mme"
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
oai_spgwc:
image: oaisoftwarealliance/oai-spgwc:latest
container_name: rfsim4g-oai-spgwc
hostname: spgwc
networks:
public_net:
environment:
TZ: Europe/Paris
SGW_INTERFACE_NAME_FOR_S11: eth0
PGW_INTERFACE_NAME_FOR_SX: eth0
DEFAULT_DNS_IPV4_ADDRESS: 172.21.3.100
DEFAULT_DNS_SEC_IPV4_ADDRESS: 8.8.4.4
PUSH_PROTOCOL_OPTION: 'true'
APN_NI_1: oai.ipv4
APN_NI_2: oai.ipv4_2
DEFAULT_APN_NI_1: oai.ipv4
UE_IP_ADDRESS_POOL_1: '12.0.0.2 - 12.0.0.254'
UE_IP_ADDRESS_POOL_2: '12.1.1.2 - 12.1.1.254'
MCC: '208'
MNC: '96'
MNC03: '096'
TAC: 1
GW_ID: 1
REALM: openairinterface.org
healthcheck:
test: /bin/bash -c "pgrep oai_spgwc"
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
oai_spgwu:
image: oaisoftwarealliance/oai-spgwu-tiny:latest
container_name: rfsim4g-oai-spgwu-tiny
hostname: spgwu
depends_on: [oai_spgwc]
networks:
public_net:
entrypoint: /bin/bash -c "/spgwuentry.sh"
volumes:
- ./spgwuentry.sh:/spgwuentry.sh
- /dev/net/tun:/dev/net/tun
cap_add:
- NET_ADMIN
- SYS_ADMIN
- NET_RAW
cap_drop:
- ALL
sysctls:
- net.ipv4.conf.eth0.rp_filter=0
- net.ipv4.conf.all.forwarding=1
- net.ipv4.conf.all.send_redirects=0
- net.ipv4.conf.default.send_redirects=0
- net.ipv4.conf.all.accept_redirects=0
- net.ipv4.conf.default.accept_redirects=0
environment:
TZ: Europe/Paris
PID_DIRECTORY: /var/run
INSTANCE: 1
SGW_INTERFACE_NAME_FOR_S1U_S12_S4_UP: eth0
PGW_INTERFACE_NAME_FOR_SGI: eth0
SGW_INTERFACE_NAME_FOR_SX: eth0
SPGWC0_HOST: spgwc
NETWORK_UE_IP: '12.0.0.0/24'
NETWORK_UE_NAT_OPTION: 'yes'
MCC: '208'
MNC: '96'
MNC03: '096'
TAC: 1
GW_ID: 1
REALM: openairinterface.org
deploy:
placement:
constraints:
- node.labels.machine == carabe
trf_gen:
image: oaisoftwarealliance/trf-gen-cn5g:focal
container_name: rfsim4g-trf-gen
hostname: trf-gen
networks:
public_net:
environment:
UPF_FQDN: 'spgwu'
USE_FQDN: 'yes'
UE_NETWORK: '12.0.0.0/24'
cap_add:
- NET_ADMIN
- SYS_ADMIN
- NET_RAW
cap_drop:
- ALL
healthcheck:
test: /bin/bash -c "ping -c 2 spgwu"
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
oai_enb0:
#image: oaisoftwarealliance/oai-enb:develop
image: porcepix.sboai.cs.eurecom.fr/oai-enb:develop-883724b9
#privileged: true
hostname: enb0
networks:
public_net:
environment:
TZ: Europe/Paris
USE_ADDITIONAL_OPTIONS: --rfsim --log_config.global_log_options level,nocolor,time
cap_add:
- ALL
volumes:
- ../../conf_files/enb.band7.25prb.rfsim.conf:/opt/oai-enb/etc/enb.conf.tmpt #adding it to template file instead
- ./enb0entry.sh:/enb0entry.sh
- /dev/cpu_dma_latency:/dev/cpu_dma_latency
- /dev/net/tun:/dev/net/tun
entrypoint: /bin/bash -c "/enb0entry.sh" ## added a custom entrypoint
healthcheck:
test: /bin/bash -c "pgrep lte-softmodem"
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
oai_ue0:
#image: oaisoftwarealliance/oai-lte-ue:develop
image: porcepix.sboai.cs.eurecom.fr/oai-lte-ue:develop-883724b9
#privileged: true
container_name: rfsim4g-oai-lte-ue0
hostname: lte-ue0
networks:
public_net:
expose:
- "10000"
environment:
TZ: Europe/Paris
USE_ADDITIONAL_OPTIONS: --rfsim -C 2680000000 -r 25 --ue-rxgain 140 --ue-txgain 120 --rfsimulator.serveraddr 192.168.61.17 --log_config.global_log_options level,nocolor,time
cap_add:
- ALL
volumes:
- ../../conf_files/lteue.usim-ci.conf:/opt/oai-lte-ue/etc/ue_usim.conf
- /dev/cpu_dma_latency:/dev/cpu_dma_latency
- /dev/net/tun:/dev/net/tun
healthcheck:
test: /bin/bash -c "pgrep lte-uesoftmodem"
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == cetautomatix
networks:
private_net:
name: rfsim4g-oai-private-net
ipam:
config:
- subnet: 192.168.68.0/26
driver_opts:
com.docker.network.bridge.name: "rfsim4g-private"
public_net:
name: rfsim4g-oai-public-net
ipam:
config:
- subnet: 192.168.61.0/26
driver_opts:
com.docker.network.bridge.name: "rfsim4g-public"
......@@ -6,7 +6,6 @@ services:
container_name: rfsim4g-cassandra
networks:
private_net:
ipv4_address: 192.168.68.2
environment:
CASSANDRA_CLUSTER_NAME: "OAI HSS Cluster"
CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch
......@@ -15,37 +14,42 @@ services:
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
db_init:
image: cassandra:2.1
container_name: rfsim4g-db-init
depends_on: [cassandra]
deploy:
restart_policy:
condition: on-failure
max_attempts: 10
networks:
private_net:
ipv4_address: 192.168.68.4
volumes:
- ./oai_db.cql:/home/oai_db.cql
entrypoint: /bin/bash -c "cqlsh --file /home/oai_db.cql 192.168.68.2 && echo 'OK'"
entrypoint: /bin/bash -c "cqlsh --file /home/oai_db.cql cassandra && echo 'OK' && sleep infinity"
deploy:
placement:
constraints:
- node.labels.machine == carabe
oai_hss:
image: oai-hss:latest
image: oaisoftwarealliance/oai-hss:latest
container_name: rfsim4g-oai-hss
hostname: hss
privileged: true
depends_on: [cassandra]
networks:
private_net:
ipv4_address: 192.168.68.3
public_net:
ipv4_address: 192.168.61.2
environment:
REALM: openairinterface.org
HSS_FQDN: hss.openairinterface.org
PREFIX: /openair-hss/etc
cassandra_Server_IP: 192.168.68.2
cassandra_Server_IP: cassandra
OP_KEY: 1006020f0a478bf6b699f15c062e42b3
LTE_K: fec86ba6eb707ed08905757b1bb44b8f
APN1: oai.ipv4
......@@ -57,32 +61,39 @@ services:
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
redis:
image: redis:6.0.5
container_name: rfsim4g-redis
hostname: redis_4g
privileged: true
networks:
public_net:
ipv4_address: 192.168.61.6
volumes:
- ./redis_extern.conf:/usr/local/etc/redis/redis.conf
entrypoint: /bin/bash -c "redis-server /usr/local/etc/redis/redis.conf"
healthcheck:
test: /bin/bash -c "redis-cli -h 192.168.61.6 -p 6380 ping"
test: /bin/bash -c "redis-cli -h redis_4g -p 6380 ping"
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
magma_mme:
image: magma-mme:latest
image: oaisoftwarealliance/magma-mme:latest
container_name: rfsim4g-magma-mme
hostname: mme
privileged: true
depends_on: [oai_hss, redis]
networks:
public_net:
ipv4_address: 192.168.61.3
environment:
TZ: Europe/Paris
REALM: openairinterface.org
......@@ -94,7 +105,7 @@ services:
FEATURES: mme_oai
volumes:
- ./mme_fd.sprint.conf:/magma-mme/etc/mme_fd.conf.tmplt
- ./mme.conf:/magma-mme/etc/mme.conf
- ./mme.conf:/magma-mme/etc/mme.conf.tmplt
- ./entrypoint.sh:/magma-mme/bin/entrypoint.sh
entrypoint: /bin/bash -c "/magma-mme/bin/entrypoint.sh"
healthcheck:
......@@ -102,15 +113,18 @@ services:
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
oai_spgwc:
image: oai-spgwc:latest
image: oaisoftwarealliance/oai-spgwc:latest
hostname: spgwc
privileged: true
depends_on: [magma_mme]
container_name: rfsim4g-oai-spgwc
networks:
public_net:
ipv4_address: 192.168.61.4
environment:
TZ: Europe/Paris
SGW_INTERFACE_NAME_FOR_S11: eth0
......@@ -134,15 +148,19 @@ services:
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
oai_spgwu:
image: oai-spgwu-tiny:latest
image: oaisoftwarealliance/oai-spgwu-tiny:latest
privileged: true
container_name: rfsim4g-oai-spgwu-tiny
depends_on: [oai_spgwc]
hostname: spgwu
networks:
public_net:
ipv4_address: 192.168.61.5
environment:
TZ: Europe/Paris
PID_DIRECTORY: /var/run
......@@ -150,7 +168,7 @@ services:
SGW_INTERFACE_NAME_FOR_S1U_S12_S4_UP: eth0
PGW_INTERFACE_NAME_FOR_SGI: eth0
SGW_INTERFACE_NAME_FOR_SX: eth0
SPGWC0_IP_ADDRESS: 192.168.61.4
SPGWC0_IP_ADDRESS: spgwc
NETWORK_UE_IP: '12.0.0.0/24'
NETWORK_UE_NAT_OPTION: 'yes'
MCC: '208'
......@@ -164,20 +182,28 @@ services:
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
trf_gen:
image: oaisoftwarealliance/trf-gen-cn5g:focal
privileged: true
container_name: rfsim4g-trf-gen
hostname: trfgen
networks:
public_net:
ipv4_address: 192.168.61.11
entrypoint: /bin/bash -c "ip route add 12.0.0.0/24 via 192.168.61.5 dev eth0; sleep infinity"
entrypoint: /bin/bash -c "ip route add 12.0.0.0/24 via spgwu dev eth0; sleep infinity"
healthcheck:
test: /bin/bash -c "ping -c 2 192.168.61.5"
test: /bin/bash -c "ping -c 2 spgwu"
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
oai_enb0:
image: oaisoftwarealliance/oai-enb:develop
......@@ -196,6 +222,10 @@ services:
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
oai_ue0:
image: oaisoftwarealliance/oai-lte-ue:develop
......@@ -216,6 +246,10 @@ services:
interval: 10s
timeout: 5s
retries: 5
deploy:
placement:
constraints:
- node.labels.machine == carabe
networks:
private_net:
......
#!/bin/bash
echo "Running the custom entry point *******************************************************************************************************************************"
echo "waiting till mme is resolved"
# Function to check if hostname is resolved
check_resolution() {
local hostname="$1"
getent hosts "$hostname" >/dev/null 2>&1
}
# Wait for hostname resolution
while ! check_resolution "mme"; do
echo "Waiting for 'mme' hostname resolution..."
sleep 1
done
# Resolve hostnames using getent
mme_ip_address_ipv4=$(getent hosts mme | awk '{print $1}')
enb_ipv4_address=$(getent hosts enb0 | awk '{print $1}')
# Resolve placeholders and create new configuration file
sed -e "s/enb0_placeholder/${enb_ipv4_address}/g" \
-e "s/mme_placeholder/${mme_ip_address_ipv4}/g" \
/opt/oai-enb/etc/enb.conf.tmpt > /opt/oai-enb/etc/enb.conf
echo "Updated config"
cat /opt/oai-enb/etc/enb.conf
# trying to execute the original entrypoint
exec /tini -v -- /opt/oai-enb/bin/entrypoint.sh /opt/oai-enb/bin/lte-softmodem -O /opt/oai-enb/etc/enb.conf
echo "Slepping to infinity"
sleep infinity
#!/bin/bash
echo "sleeping for all the others NFs to be ready"
sleep 3
set -euo pipefail
echo "error point 1"
# we need to resolve
HSS_IP_ADDRESS=`getent hosts hss | awk '{print $1}'`
echo "error point 2"
MME_IP_ADDRESS=`getent hosts mme | awk '{print $1}'`
echo "error point 3"
REDIS_IP_ADDRESS=`getent hosts redis_4g | awk '{print $1}'`
echo "error point 4"
SPGWC_IP_ADDRESS=`getent hosts spgwc | awk '{print $1}'`
echo "error point 5"
set +euo
echo "error point 6"
echo "all NFs should be resolved"
echo $HSS_IP_ADDRESS
echo $MME_IP_ADDRESS
echo $REDIS_IP_ADDRESS
echo $SPGWC_IP_ADDRESS
INSTANCE=1
PREFIX='/magma-mme/etc'
MY_REALM='openairinterface.org'
......@@ -7,24 +34,34 @@ MY_REALM='openairinterface.org'
declare -A MME_CONF
pushd $PREFIX
MME_CONF[@MME_S6A_IP_ADDR@]="192.168.61.3"
MME_CONF[@MME_S6A_IP_ADDR@]="$MME_IP_ADDRESS"
MME_CONF[@INSTANCE@]=$INSTANCE
MME_CONF[@PREFIX@]=$PREFIX
MME_CONF[@REALM@]=$MY_REALM
MME_CONF[@MME_FQDN@]="mme.${MME_CONF[@REALM@]}"
MME_CONF[@HSS_HOSTNAME@]='hss'
MME_CONF[@HSS_FQDN@]="${MME_CONF[@HSS_HOSTNAME@]}.${MME_CONF[@REALM@]}"
MME_CONF[@HSS_IP_ADDR@]="192.168.61.2"
MME_CONF[@HSS_IP_ADDR@]="$HSS_IP_ADDRESS"
MME_CONF[@SPGWC_IP_ADDR@]="$SPGWC_IP_ADDRESS"
MME_CONF[@REDIS_IP_ADDR@]="$REDIS_IP_ADDRESS"
cp mme_fd.conf.tmplt $PREFIX/mme_fd.conf
cp $PREFIX/mme_fd.conf.tmplt $PREFIX/mme_fd.conf
cp $PREFIX/mme.conf.tmplt $PREFIX/mme.conf
for K in "${!MME_CONF[@]}"; do
egrep -lRZ "$K" $PREFIX/mme_fd.conf | xargs -0 -l sed -i -e "s|$K|${MME_CONF[$K]}|g"
ret=$?;[[ ret -ne 0 ]] && echo "Could not replace $K with ${MME_CONF[$K]}"
egrep -lRZ "$K" $PREFIX/mme.conf | xargs -0 -l sed -i -e "s|$K|${MME_CONF[$K]}|g"
ret=$?;[[ ret -ne 0 ]] && echo "Could not replace $K with ${MME_CONF[$K]}"
done
sed -i -e "s@etc/freeDiameter@etc@" /magma-mme/etc/mme_fd.conf
sed -i -e "s@bind: 127.0.0.1@bind: 192.168.61.6@" /etc/magma/redis.yml
sed -i -e "s@bind: 127.0.0.1@bind: $REDIS_IP_ADDRESS@" /etc/magma/redis.yml
cat /magma-mme/etc/mme_fd.conf
cat $PREFIX/mme.conf
cat /etc/magma/redis.yml
# Generate freeDiameter certificate
popd
cd /magma-mme/scripts
......@@ -33,4 +70,8 @@ cd /magma-mme/scripts
cd /magma-mme
nohup /magma-mme/bin/sctpd > /var/log/sctpd.log 2>&1 &
sleep 5
/magma-mme/bin/oai_mme -c /magma-mme/etc/mme.conf
/magma-mme/bin/oai_mme -c /magma-mme/etc/mme.conf || true
sleep 5
cat /var/log/mme.log
sleep infinity
......@@ -98,9 +98,9 @@ MME :
NETWORK_INTERFACES :
{
MME_INTERFACE_NAME_FOR_S1_MME = "eth0";
MME_IPV4_ADDRESS_FOR_S1_MME = "192.168.61.3/24";
MME_IPV4_ADDRESS_FOR_S1_MME = "@MME_S6A_IP_ADDR@/24";
MME_INTERFACE_NAME_FOR_S11_MME = "eth0";
MME_IPV4_ADDRESS_FOR_S11_MME = "192.168.61.3/24";
MME_IPV4_ADDRESS_FOR_S11_MME = "@MME_S6A_IP_ADDR@/24";
MME_PORT_FOR_S11_MME = 2123;
};
......@@ -129,6 +129,6 @@ MME :
S-GW :
{
SGW_IPV4_ADDRESS_FOR_S11 = "192.168.61.4";
SGW_IPV4_ADDRESS_FOR_S11 = "@SPGWC_IP_ADDR@";
};
};
......@@ -12,7 +12,7 @@
# TODO: make sensible, production-aware config decisions
bind 192.168.61.6
bind redis_4g
port 6380
daemonize no
......
#!/bin/bash
#export SPGWC0_IP_ADDRESS=`getent hosts $SPGWC0_HOST | awk '{print $1}'`
export SPGWC0_IP_ADDRESS=`getent hosts spgwc | awk '{print $1}'`
echo $SPGWC0_IP_ADDRESS
# Enabling NET_ADMIN and SYS_ADMIN for SPGWU executable
ls -lst /openair-spgwu-tiny/bin/oai_spgwu
getcap /openair-spgwu-tiny/bin/oai_spgwu
echo "changing capabilities on oai_spgwu"
#setcap 'cap_net_admin=ep cap_sys_admin=ep' /openair-spgwu-tiny/bin/oai_spgwu
echo "checking capabilities on oai_spgwu"
ls -lst /openair-spgwu-tiny/bin/oai_spgwu
getcap /openair-spgwu-tiny/bin/oai_spgwu
# Command to run the Python entrypoint script
python3 /openair-spgwu-tiny/bin/entrypoint.py /openair-spgwu-tiny/bin/oai_spgwu -c /openair-spgwu-tiny/etc/spgw_u.conf -o
# If the user does not specify a command at runtime and ENTRYPOINT is defined, the command specified by ENTRYPOINT is executed, with CMD as its arguments.
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment