Merge branch 'framed_routing_ebpf' into 'develop'

Add Framed Routing for Simple Switch and eBPF

See merge request oai/cn5g/oai-cn5g-upf!57

common-src @ dcb1fdbd

-Subproject commit 548d88125b345920cbeee4c9db7cccbeaa6579ce
+Subproject commit dcb1fdbdbd701f6e16c366fe24ae1e809c32391b

src/oai_upf/CMakeLists.txt

@@ -285,6 +285,8 @@ ADD_SUBDIRECTORY(${SRC_TOP_DIR}/upf_app/simpleswitch ${CMAKE_CURRENT_BINARY_DIR}
 ADD_SUBDIRECTORY(${SRC_TOP_DIR}/gtpv1u ${CMAKE_CURRENT_BINARY_DIR}/gtpv1u)
 ADD_SUBDIRECTORY(${SRC_TOP_DIR}/udp ${CMAKE_CURRENT_BINARY_DIR}/udp)
 ADD_SUBDIRECTORY(${SRC_TOP_DIR}/upf_app/interfaces ${CMAKE_CURRENT_BINARY_DIR}/interfaces)
+ADD_SUBDIRECTORY(${SRC_TOP_DIR}/upf_app/framed_routing ${CMAKE_CURRENT_BINARY_DIR}/framed_routing)
+
 ################################################################################
 # Specific part for oai_upf folder
 
@@ -300,10 +302,12 @@ include_directories(${SRC_TOP_DIR}/pfcp)
 include_directories(${SRC_TOP_DIR}/udp)
 include_directories(${SRC_TOP_DIR}/upf_app)
 include_directories(${SRC_TOP_DIR}/upf_app/bpf)
+include_directories(${SRC_TOP_DIR}/upf_app/bpf/rules/pdr)
 include_directories(${SRC_TOP_DIR}/upf_app/bpf/rules/include)
 include_directories(${SRC_TOP_DIR}/upf_app/app)
 include_directories(${SRC_TOP_DIR}/upf_app/interfaces)
 include_directories(${SRC_TOP_DIR}/upf_app/simpleswitch)
+include_directories(${SRC_TOP_DIR}/upf_app/framed_routing)
 include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/3gpp)
 include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/common)
 include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/model/common_model)
@@ -342,7 +346,7 @@ include(${SRC_TOP_DIR}/${MOUNTED_COMMON}/utils/utils.cmake)
 # folly glog dl double-conversion for FB folly library
 target_link_libraries (upf 
   ${ASAN}  -Wl,--start-group CN_UTILS
-  ${SRC_TOP_DIR}/../build/ext/libbpf/src/libbpf.so 
+  ${SRC_TOP_DIR}/../build/ext/libbpf/src/libbpf.so
   APP
   SIMPLE_SWITCH
   interfaces
@@ -356,7 +360,7 @@ target_link_libraries (upf
   UPF_XDP
   UPF_TC
   gflags glog dl double-conversion folly fmt spdlog -Wl,--end-group
-  pthread m rt config++ 
+  pthread m rt config++
   event boost_system curl yaml-cpp cpr::cpr
 )
 

src/oai_upf/main.cpp

@@ -26,6 +26,7 @@
 #include "upf_config.hpp"
 #include "upf_config_yaml.hpp"
 #include "sbi_helper.hpp"
+#include <pfcp_session_lookup_xdp_user.h>
 
 #include <boost/asio.hpp>
 #include <iostream>
@@ -124,6 +125,10 @@ void setup_bpf() {
   Logger::upf_app().info("UDP interface: %s", sUDPInterface.c_str());
 
   UserPlaneComponent::getInstance().setup(sGTPInterface, sUDPInterface);
+
+  auto pPFCP_Session_LookupProgram =
+      UserPlaneComponent::getInstance().getPFCP_Session_LookupProgram();
+  pPFCP_Session_LookupProgram->setFramedRouting(upf_cfg.enable_fr);
 }
 
 //------------------------------------------------------------------------------

src/upf_app/SessionManager.cpp

@@ -309,7 +309,7 @@ void SessionManager::createSessionDirection(
   if (!pdrs.empty()) {
     auto pdrHighPrecedence = pdrs.front();
     logger.debug(
-        "The PDR %d has the Highest Precedence", direction,
+        "The %s PDR %u has the Highest Precedence", direction,
         pdrHighPrecedence->pdr_id.rule_id);
     if (direction == "Uplink") {
       createBPFSessionUL(pSession_establishment, pdrHighPrecedence);
@@ -401,6 +401,18 @@ void SessionManager::processPDRDetails(
 
   std::vector<std::shared_ptr<pfcp::pfcp_qer>> pQer;
 
+  if (upf_cfg.enable_fr && direction == "Downlink") {
+    if (ueIpAddress.v4) {
+      std::vector<pfcp::framed_route_t> framedRoutes;
+      if (pdi.get(framedRoutes)) {
+        SessionProgramManager::getInstance().addFramedRoutes(
+            ueIpAddress.ipv4_address.s_addr, framedRoutes);
+      }
+    } else {
+      Logger::upf_app().warn("Framed Route is not yet supported for Ipv6");
+    }
+  }
+
   if (upf_cfg.enable_qos) {
     pQer = (direction == "Uplink") ? pSession->qers_uplink :
                                      pSession->qers_downlink;
@@ -458,7 +470,7 @@ void SessionManager::updateBPFSession(
 
       auto pdrHighPrecedenceDl = pSession->pdrs_downlink[0];
       Logger::upf_app().debug(
-          "The Downlink PDR %d has the Highest Precedence",
+          "The Downlink PDR %u has the Highest Precedence",
           pdrHighPrecedenceDl->pdr_id.rule_id);
 
       Logger::upf_app().debug(
@@ -475,7 +487,7 @@ void SessionManager::updateBPFSession(
 
       auto pdrHighPrecedenceUl = pSession->pdrs_uplink[0];
       Logger::upf_app().debug(
-          "The Uplink PDR %d has the Highest Precedence",
+          "The Uplink PDR %u has the Highest Precedence",
           pdrHighPrecedenceUl->pdr_id.rule_id);
 
       Logger::upf_app().debug(
@@ -486,10 +498,33 @@ void SessionManager::updateBPFSession(
     }
   }
 
+  // TODO: Update_pdrs (update Framed Routes Map)
+
   for (auto it : mod_req->pfcp_ies.remove_pdrs) {
     Logger::upf_app().debug("Delete PDRs");
     Logger::upf_app().debug(
         "PDRs and FARs map entries are obsolete and need to be deleted");
+
+    pfcp::pdr_id_t pdr_id;
+    if (it.get(pdr_id)) {
+      Logger::upf_app().debug("Remove PDR with id %u", pdr_id.rule_id);
+      for (auto pdr : pSession->pdrs) {
+        if (pdr_id.rule_id == pdr->pdr_id.rule_id) {
+          Logger::upf_app().debug(
+              "Found PDR with id %u in list 'pdrs'", pdr_id.rule_id);
+          if (upf_cfg.enable_fr) {
+            pfcp::pdi pdi;
+            if (pdr->get(pdi)) {
+              std::vector<pfcp::framed_route_t> framedRoutes;
+              if (pdi.get(framedRoutes)) {
+                SessionProgramManager::getInstance().removeFramedRoutes(
+                    framedRoutes);
+              }
+            }
+          }
+        }
+      }
+    }
   }
 }
 
@@ -578,6 +613,18 @@ void SessionManager::updateBPFSessionDL(
   // pSession->qerIDsPerPDR.qers;
   // std::vector<std::shared_ptr<pfcp::pfcp_qer>> pQer = pSession->qers;
 
+  if (upf_cfg.enable_fr) {
+    if (ueIpAddress.v4) {
+      std::vector<pfcp::framed_route_t> framedRoutes;
+      if (pdi.get(framedRoutes)) {
+        SessionProgramManager::getInstance().addFramedRoutes(
+            ueIpAddress.ipv4_address.s_addr, framedRoutes);
+      }
+    } else {
+      Logger::upf_app().warn("Framed Route is not yet supported for Ipv6");
+    }
+  }
+
   if (teid_ul) {
     SessionProgramManager::getInstance().createPipeline(
         seidul, fteid.teid, INTERFACE_VALUE_CORE,
@@ -604,6 +651,19 @@ void SessionManager::removeBPFSession(
     // throw std::runtime_error("Session Does Not Exist. It Cannot be Removed");
   }
 
+  if (upf_cfg.enable_fr) {
+    // Remove framed route to ue_ip mapping
+    for (auto pdr : pSession->pdrs) {
+      pfcp::pdi pdi;
+      if (pdr->get(pdi)) {
+        std::vector<pfcp::framed_route_t> framedRoutes;
+        if (pdi.get(framedRoutes)) {
+          SessionProgramManager::getInstance().removeFramedRoutes(framedRoutes);
+        }
+      }
+    }
+  }
+
   SessionProgramManager::getInstance().removePipeline(seid);
   Logger::upf_app().debug("Session 0x%x Has Been Removed Successfully", seid);
 }

src/upf_app/SessionProgramManager.cpp

@@ -6,6 +6,8 @@
 #include <pfcp_session_lookup_xdp_user.h>
 #include <UserPlaneComponent.h>
 #include <net/if.h>  // if_nametoindex
+#include <framed_routing/FramedRouting.hpp>
+#include <framed_routing_bpf.h>
 
 #include <observer/OnStateChangeSessionProgramObserver.h>
 #include <spdlog/fmt/ostr.h>
@@ -301,7 +303,48 @@ uint32_t SessionProgramManager::getGnodebIp(
   return gNBIpAddress.ipv4_address.s_addr;
 }
 
-//---------------------------------------------------------------------------------------------------------------
+void SessionProgramManager::addFramedRoutes(
+    uint32_t ueIpAddress,
+    const std::vector<pfcp::framed_route_t>& framedRoutes) {
+  auto pPFCP_Session_LookupProgram =
+      UserPlaneComponent::getInstance().getPFCP_Session_LookupProgram();
+
+  for (const auto& framedRoute : framedRoutes) {
+    Logger::upf_app().info(
+        "Add framed route to ue_ip mapping %s to UE IP 0x%x",
+        framedRoute.framed_route, ueIpAddress);
+    std::stringstream ss(framedRoute.framed_route);
+    std::string ipsubnetmask;
+    while (std::getline(ss, ipsubnetmask, ' ')) {
+      std::pair<uint32_t, uint32_t> ipCidr =
+          fr::FramedRouting::extractIPCidr(ipsubnetmask);
+      auto key = framed_routing_key_for_ip_cidr(ipCidr.first, ipCidr.second);
+      pPFCP_Session_LookupProgram->updateFramedRouteMappingMap(
+          ueIpAddress, key);
+    }
+  }
+}
+
+void SessionProgramManager::removeFramedRoutes(
+    const std::vector<pfcp::framed_route_t>& framedRoutes) {
+  auto pPFCP_Session_LookupProgram =
+      UserPlaneComponent::getInstance().getPFCP_Session_LookupProgram();
+  for (const auto& framedRoute : framedRoutes) {
+    std::stringstream ss(framedRoute.framed_route);
+    std::string ipsubnetmask;
+    Logger::upf_app().info(
+        "Remove framed route to ue_ip mapping for %s",
+        framedRoute.framed_route);
+    while (std::getline(ss, ipsubnetmask, ' ')) {
+      std::pair<uint32_t, uint32_t> ipCidr =
+          fr::FramedRouting::extractIPCidr(ipsubnetmask);
+      auto key = framed_routing_key_for_ip_cidr(ipCidr.first, ipCidr.second);
+      pPFCP_Session_LookupProgram->removeFramedRoute(key);
+    }
+  }
+}
+
+/*---------------------------------------------------------------------------------------------------------------*/
 // Function to create a pipeline for a given session and FAR
 void SessionProgramManager::createPipeline(
     uint64_t seid, uint32_t teid1, uint8_t sourceInterface,

src/upf_app/SessionProgramManager.h

@@ -107,6 +107,15 @@ class SessionProgramManager {
   /*---------------------------------------------------------------------------------------------------------------*/
   pfcp_far_t_ createFar(std::shared_ptr<pfcp::pfcp_far> pFar);
 
+  /*---------------------------------------------------------------------------------------------------------------*/
+  void addFramedRoutes(
+      uint32_t ueIpAddress,
+      const std::vector<pfcp::framed_route_t>& framedRoutes);
+
+  /*---------------------------------------------------------------------------------------------------------------*/
+  void removeFramedRoutes(
+      const std::vector<pfcp::framed_route_t>& framedRoutes);
+
   /*---------------------------------------------------------------------------------------------------------------*/
   void createPipeline(
       uint64_t seid, uint32_t teid1, uint8_t sourceInterface,

src/upf_app/app/upf_config.hpp

@@ -120,6 +120,8 @@ class upf_config {
   } nf_addr;
 
   bool enable_snat;
+  bool enable_fr;
+
   std::vector<pdn_cfg_t> pdns;
   std::vector<pfcp::node_id_t> smfs;
 
@@ -152,6 +154,7 @@ class upf_config {
         max_pfcp_sessions(100),
         nsf(),
         enable_snat(false),
+        enable_fr(false),
         nrf_addr() {
     itti.itti_timer_sched_params.sched_priority = 85;
     itti.n3_sched_params.sched_priority         = 84;

src/upf_app/app/upf_config_yaml.cpp

@@ -32,7 +32,8 @@ namespace oai::config {
 
 //------------------------------------------------------------------------------
 upf_support_features::upf_support_features(
-    bool enable_bpf_datapath, bool enable_qos, bool enable_snat) {
+    bool enable_bpf_datapath, bool enable_qos, bool enable_snat,
+    bool enable_fr) {
   m_config_name = "Supported Features";
 
   m_enable_bpf_datapath = option_config_value(
@@ -43,6 +44,8 @@ upf_support_features::upf_support_features(
 
   m_enable_snat = option_config_value(
       UPF_CONFIG_SUPPORT_FEATURES_ENABLE_SNAT_LABEL, enable_snat);
+  m_enable_fr =
+      option_config_value(UPF_CONFIG_SUPPORT_FEATURES_ENABLE_FR, enable_fr);
 }
 
 //------------------------------------------------------------------------------
@@ -59,6 +62,8 @@ void upf_support_features::from_yaml(const YAML::Node& node) {
   if (node[UPF_CONFIG_SUPPORT_FEATURES_ENABLE_SNAT]) {
     m_enable_snat.from_yaml(node[UPF_CONFIG_SUPPORT_FEATURES_ENABLE_SNAT]);
   }
+  if (node[UPF_CONFIG_SUPPORT_FEATURES_ENABLE_FR])
+    m_enable_fr.from_yaml(node[UPF_CONFIG_SUPPORT_FEATURES_ENABLE_FR]);
 }
 
 //------------------------------------------------------------------------------
@@ -90,6 +95,13 @@ std::string upf_support_features::to_string(const std::string& indent) const {
   out.append(indent).append(fmt::format(
       BASE_FORMATTER, INNER_LIST_ELEM,
       UPF_CONFIG_SUPPORT_FEATURES_ENABLE_SNAT_LABEL, inner_width, enable_snat));
+
+  std::string enable_fr = m_enable_fr.get_value() ? UPF_CONFIG_OPTION_YES_STR :
+                                                    UPF_CONFIG_OPTION_NO_STR;
+
+  out.append(indent).append(fmt::format(
+      BASE_FORMATTER, INNER_LIST_ELEM, UPF_CONFIG_SUPPORT_FEATURES_ENABLE_FR,
+      inner_width, enable_fr));
   return out;
 }
 
@@ -98,7 +110,7 @@ upf::upf(
     const std::string& name, const std::string& host, const sbi_interface& sbi,
     const std::map<std::string, upf_interface_config>& interfaces)
     : nf(name, host, sbi),
-      m_upf_support_features(false, false, false),
+      m_upf_support_features(false, false, false, false),
       m_interfaces(interfaces) {
   model::nrf::SnssaiUpfInfoItem item;
   item.setSNssai(DEFAULT_SNSSAI);
@@ -209,6 +221,11 @@ bool upf_support_features::get_option_enable_snat() const {
   return m_enable_snat.get_value();
 }
 
+//------------------------------------------------------------------------------
+bool upf_support_features::get_option_enable_fr() const {
+  return m_enable_fr.get_value();
+}
+
 //------------------------------------------------------------------------------
 const upf_support_features& upf::get_support_features() const {
   return m_upf_support_features;
@@ -387,6 +404,7 @@ void upf_config_yaml::to_upf_config(upf_config& cfg) {
       upf_local->get_support_features().get_option_enable_bpf_datapath();
   cfg.enable_qos  = upf_local->get_support_features().get_option_enable_qos();
   cfg.enable_snat = upf_local->get_support_features().get_option_enable_snat();
+  cfg.enable_fr   = upf_local->get_support_features().get_option_enable_fr();
 
   auto snssai_upf_list = upf_local->get_upf_info().getSNssaiUpfInfoList();
   for (const auto& snssai : snssai_upf_list) {

src/upf_app/app/upf_config_yaml.hpp

@@ -37,6 +37,7 @@ constexpr auto UPF_CONFIG_SUPPORT_FEATURES_ENABLE_BPF = "enable_bpf_datapath";
 constexpr auto UPF_CONFIG_SUPPORT_FEATURES_ENABLE_BPF_LABEL =
     "Enable BPF Datapath";
 constexpr auto UPF_CONFIG_SUPPORT_FEATURES_ENABLE_SNAT       = "enable_snat";
+constexpr auto UPF_CONFIG_SUPPORT_FEATURES_ENABLE_FR         = "enable_fr";
 constexpr auto UPF_CONFIG_SUPPORT_FEATURES_ENABLE_SNAT_LABEL = "Enable SNAT";
 
 constexpr auto UPF_CONFIG_SUPPORT_FEATURES_ENABLE_QOS       = "enable_qos";
@@ -57,10 +58,12 @@ class upf_support_features : public config_type {
   option_config_value m_enable_bpf_datapath{};
   option_config_value m_enable_qos{};
   option_config_value m_enable_snat{};
+  option_config_value m_enable_fr{};
 
  public:
   explicit upf_support_features(
-      bool enable_bpf_datapath, bool enable_qos, bool enable_snat);
+      bool enable_bpf_datapath, bool enable_qos, bool enable_snat,
+      bool enable_fr);
 
   void from_yaml(const YAML::Node& node) override;
 
@@ -68,6 +71,7 @@ class upf_support_features : public config_type {
   [[nodiscard]] bool get_option_enable_bpf_datapath() const;
   [[nodiscard]] bool get_option_enable_qos() const;
   [[nodiscard]] bool get_option_enable_snat() const;
+  [[nodiscard]] bool get_option_enable_fr() const;
 };
 
 class upf_interface_config : public local_interface {

src/upf_app/app/upf_n4.cpp

@@ -237,7 +237,7 @@ upf_n4::upf_n4()
   up_function_features.udbc  = 0;
   up_function_features.quoac = 0;
   up_function_features.trace = 0;
-  up_function_features.frrt  = 0;
+  up_function_features.frrt  = static_cast<uint16_t>(upf_cfg.enable_fr);
 
   enterprise_specific.enterprise_id    = OAI_PRIVATE_ENTERPRISE_NUMBER;
   enterprise_specific.proprietary_data = "OAI UPF";

src/upf_app/bpf/rules/include/pfcp_session_lookup_maps.h

@@ -73,5 +73,20 @@ struct {
 } m_session_mapping SEC(".maps");
 
 /*---------------------------------------------------------------------------------------------------------------*/
+/* Framed Routing */
+
+struct {
+  __uint(type, BPF_MAP_TYPE_HASH);
+  __uint(max_entries, MAX_UEs);
+  __type(key, u32);    // hash_framed_routing_key
+  __type(value, u32);  // ue_ip
+} m_framed_route_mapping SEC(".maps");
+
+struct {
+  __uint(type, BPF_MAP_TYPE_HASH);
+  __uint(max_entries, 1);  // Single entry for the flag
+  __type(key, u8);         // Key is a constant, e.g., 0
+  __type(value, u8);       // Value indicates if framed routing is enabled
+} framed_routing_flag SEC(".maps");
 
 #endif  // __PFCP_SESSION_LOOKUP_MAPS_H__

src/upf_app/bpf/rules/pdr/framed_routing_bpf.h

+#ifndef OPENAIRINTERFACE_FRAMED_ROUTING_BPF_H
+#define OPENAIRINTERFACE_FRAMED_ROUTING_BPF_H
+
+#include <types.h>
+#include <stdint.h>
+
+struct FramedRoutingKeyBPF {
+  uint32_t networkAddress;
+  uint32_t subnet;
+};
+
+static __always_inline uint32_t
+hash_framed_routing_key(struct FramedRoutingKeyBPF* key) {
+  uint32_t hash = 17;
+  hash          = hash * 31 + key->networkAddress;
+  hash          = hash ^ key->subnet;
+  return hash;
+}
+
+static __always_inline struct FramedRoutingKeyBPF
+framed_routing_key_for_ip_cidr(uint32_t ip, uint32_t cidr) {
+  const uint32_t ipv4Size = 32;
+  // Calculate the subnet address
+  uint32_t subnet_mask = 0xffffffff << (ipv4Size - cidr);
+  // Calculate the network address
+  uint32_t network_address = subnet_mask & ip;
+
+  struct FramedRoutingKeyBPF key = {network_address, subnet_mask};
+
+  return key;
+}
+
+#endif  // OPENAIRINTERFACE_FRAMED_ROUTING_BPF_H

src/upf_app/bpf/rules/pdr/pfcp_session_lookup_xdp_kernel.c

@@ -19,6 +19,7 @@
 #include <utils/logger.h>
 #include <utils/utils.h>
 #include <next_prog_rule_key.h>
+#include <framed_routing_bpf.h>
 
 #ifdef KERNEL_SPACE
 #include <linux/in.h>
@@ -54,6 +55,33 @@ static __always_inline u32 tail_call_next_prog(
     bpf_tail_call(ctx, &m_next_rule_prog, *index_prog);
   }
 
+  // Framed Routing
+  u8 key                     = 0;  // Key is 0 since we only have one flag
+  u8* framed_routing_enabled = bpf_map_lookup_elem(&framed_routing_flag, &key);
+  if (framed_routing_enabled && *framed_routing_enabled) {
+    // check if it is a framed route address, and if yes try again the lookup
+    // with the mapped address
+    uint32_t big_endian_ue_ip = __builtin_bswap32(ipv4_address);
+#pragma clang loop unroll(full)
+    for (uint32_t i = 32; i > 0; i--) {
+      struct FramedRoutingKeyBPF key =
+          framed_routing_key_for_ip_cidr(big_endian_ue_ip, i);
+      uint32_t fr_key = hash_framed_routing_key(&key);
+      uint32_t* fr_ue_ip =
+          bpf_map_lookup_elem(&m_framed_route_mapping, &fr_key);
+      if (fr_ue_ip) {
+        bpf_debug("Uplink: found ip  0x%x", fr_ue_ip);
+        map_key.ipv4_address = (u32) *fr_ue_ip;
+        index_prog = bpf_map_lookup_elem(&m_next_rule_prog_index, &map_key);
+        if (index_prog) {
+          bpf_debug(
+              "Value of the eBPF tail call, index_prog = %d", *index_prog);
+          bpf_tail_call(ctx, &m_next_rule_prog, *index_prog);
+        }
+      }
+    }
+  }
+
   bpf_debug("BPF tail call was not executed!");
   bpf_debug("Check your key and its endianess");
 
@@ -75,6 +103,32 @@ handle_downlink_traffic(struct xdp_md* ctx, u32 ue_ip_address) {
     tail_call_next_prog(ctx, teid_dl, INTERFACE_VALUE_CORE, ue_ip_address);
   }
 
+  // Framed Routing
+  u8 key                     = 0;  // Key is 0 since we only have one flag
+  u8* framed_routing_enabled = bpf_map_lookup_elem(&framed_routing_flag, &key);
+  if (framed_routing_enabled && *framed_routing_enabled) {
+    // check if it is a framed route address, and if yes try again the lookup
+    // with the mapped address
+    uint32_t big_endian_ue_ip = __builtin_bswap32(ue_ip_address);
+    for (uint32_t i = 32; i > 0; i--) {
+      struct FramedRoutingKeyBPF key =
+          framed_routing_key_for_ip_cidr(big_endian_ue_ip, i);
+      uint32_t fr_key = hash_framed_routing_key(&key);
+      uint32_t* fr_ue_ip =
+          bpf_map_lookup_elem(&m_framed_route_mapping, &fr_key);
+      if (fr_ue_ip) {
+        session = bpf_map_lookup_elem(&m_session_mapping, fr_ue_ip);
+        if (session) {
+          u32 teid_dl = session->teid_dl;
+          bpf_debug(
+              "TEID downlink: 0x%x was found for Framed Route IP: 0x%x",
+              teid_dl, big_endian_ue_ip);
+          tail_call_next_prog(ctx, teid_dl, INTERFACE_VALUE_CORE, *fr_ue_ip);
+        }
+      }
+    }
+  }
+
   bpf_debug("BPF tail call was not executed!");
 
   return XDP_PASS;

src/upf_app/framed_routing/CMakeLists.txt

+
+include_directories(${SRC_TOP_DIR}/common)
+include_directories(${SRC_TOP_DIR}/common/msg)
+include_directories(${SRC_TOP_DIR}/common/utils)
+include_directories(${SRC_TOP_DIR}/itti)
+include_directories(${SRC_TOP_DIR}/gtpv1u)
+include_directories(${SRC_TOP_DIR}/oai_upf)
+include_directories(${SRC_TOP_DIR}/upf_app)
+include_directories(${SRC_TOP_DIR}/upf_app/app)
+include_directories(${SRC_TOP_DIR}/upf_app/bpf)
+include_directories(${SRC_TOP_DIR}/upf_app/include)
+include_directories(${SRC_TOP_DIR}/upf_app/simpleswitch)
+include_directories(${SRC_TOP_DIR}/pfcp)
+include_directories(${SRC_TOP_DIR}/udp)
+include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/logger)
+include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/common)
+include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/config)
+include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/model/common_model)
+include(${SRC_TOP_DIR}/${MOUNTED_COMMON}/model/nrf/nrf_model.cmake)
+include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/utils)
+include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/utils/bstr)
+
+add_library(FRAMED_ROUTING STATIC
+        FramedRouting.cpp
+        LocalRouting.cpp
+        LocalRouting.hpp
+)
+
+

src/upf_app/framed_routing/FramedRouting.cpp

+//
+// Created by root on 5/10/24.
+//
+
+#include "FramedRouting.hpp"
+#include <utility>
+#include <sstream>
+#include <iostream>
+#include <upf_config.hpp>
+
+extern oai::config::upf_config upf_cfg;
+
+namespace fr {
+
+FramedRouting::FramedRouting(std::shared_ptr<LocalRouting> localRouting)
+    : localRouting(std::move(localRouting)) {}
+
+void FramedRouting::addFramedRoute(
+    const uint32_t ue_ip, const pfcp::framed_route_s& framed_route_s) {
+  std::stringstream ss(framed_route_s.framed_route);
+  std::string ipsubnetmask;
+  while (std::getline(ss, ipsubnetmask, this->pdi_fr_delimeter)) {
+    std::pair<uint32_t, uint32_t> ipCidr = this->extractIPCidr(ipsubnetmask);
+    auto key                             = createFramedRoutingKey(ipCidr);
+    auto routing_info = createLocalRoutingInformation(ipCidr);
+    auto snat_info    = createLocalSnatInformation(ipCidr);
+    this->KeyToIp.insert({key, ue_ip});
+    localRouting->add_route(routing_info);
+    localRouting->add_source_snat(snat_info);
+  }
+}
+
+uint32_t FramedRouting::retrieveUEIp(const uint32_t destination_ip) const {
+  for (uint32_t i = 32; i > 0; --i) {
+    FramedRoutingKey framedRoutingKey =
+        createFramedRoutingKey({destination_ip, i});
+
+    auto ip = this->KeyToIp.find(framedRoutingKey);
+
+    if (ip != KeyToIp.end()) {
+      return ip->second;
+    };
+  };
+  return 0;
+};
+
+uint32_t FramedRouting::retrieveUEIp(
+    const pfcp::framed_route_s& framed_route_s) const {
+  std::stringstream ss(framed_route_s.framed_route);
+  std::string ipsubnetmask;
+
+  while (std::getline(ss, ipsubnetmask, this->pdi_fr_delimeter)) {
+    std::pair<uint32_t, uint32_t> ipCidr = this->extractIPCidr(ipsubnetmask);
+    FramedRoutingKey framedRoutingKey =
+        createFramedRoutingKey({ipCidr.first, ipCidr.second});
+    auto ip = this->KeyToIp.find(framedRoutingKey);
+    if (ip != KeyToIp.end()) {
+      return ip->second;
+    }
+  }
+  return 0;
+}
+void FramedRouting::remove_entry(uint32_t ue_ip) {
+  for (uint32_t i = 32; i > 0; --i) {
+    FramedRoutingKey framedRoutingKey = createFramedRoutingKey({ue_ip, i});
+    auto ip                           = this->KeyToIp.find(framedRoutingKey);
+    if (ip != KeyToIp.end()) {
+      this->KeyToIp.erase(ip);
+      this->localRouting->delete_route(ip->second);
+    };
+  };
+}
+
+std::pair<uint32_t, uint32_t> FramedRouting::extractIPCidr(
+    const std::string& fr_subnet) {
+  const char subnet_delimeter = '/';
+  std::string ipSubnet        = fr_subnet;
+  uint32_t ip                 = 0;
+  uint32_t cidr               = 0;
+
+  const std::string ip_substring =
+      ipSubnet.substr(0, ipSubnet.find(subnet_delimeter));
+  ip = framedIPToUeIP(ip_substring);
+
+  std::reverse(ipSubnet.begin(), ipSubnet.end());
+  std::string subnet_substring =
+      ipSubnet.substr(0, ipSubnet.rfind(subnet_delimeter));
+  cidr = frameSubnetToUInt(subnet_substring);
+  return std::pair<uint32_t, uint32_t>{ip, cidr};
+}
+
+FramedRoutingKey FramedRouting::createFramedRoutingKey(
+    const std::pair<uint32_t, uint32_t> ipCidr) const {
+  const uint32_t ipv4Size      = 32;
+  const uint32_t subnet_adress = 0xffffffff << (ipv4Size - ipCidr.second);
+  const uint32_t networkAdress = subnet_adress & ipCidr.first;
+  return FramedRoutingKey{networkAdress, subnet_adress};
+}
+
+RoutingInformation FramedRouting::createLocalRoutingInformation(
+    const std::pair<uint32_t, uint32_t>& ipCidr) const {
+  struct in_addr addr;
+  addr.s_addr             = htonl(ipCidr.first);
+  std::string destination = inet_ntoa(addr);
+  uint32_t netmask        = ipCidr.second;
+  std::string device      = "tun0";
+  // todo (phine.tech) use correct DNN
+  struct in_addr address4_gw = {};
+  address4_gw.s_addr         = upf_cfg.pdns[0].network_ipv4.s_addr + be32toh(1);
+  std::string gateway_address = inet_ntoa(address4_gw);
+
+  return fr::RoutingInformation{destination, netmask, device, gateway_address};
+}
+
+SourceNatInformation FramedRouting::createLocalSnatInformation(
+    const std::pair<uint32_t, uint32_t>& ipCidr) const {
+  struct in_addr addr;
+  addr.s_addr              = htonl(ipCidr.first);
+  std::string destination  = inet_ntoa(addr);
+  uint32_t netmask         = ipCidr.second;
+  std::string device       = upf_cfg.n6.if_name;
+  std::string snat_address = oai::utils::conv::toString(upf_cfg.n6.addr4);
+
+  return SourceNatInformation{destination, netmask, device, snat_address};
+}
+
+}  // namespace fr
\ No newline at end of file

src/upf_app/framed_routing/FramedRouting.hpp

+//
+// Created by root on 5/10/24.
+//
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+#include "FramedRoutingHash.h"
+#include "pfcp_pdr.hpp"
+#include "LocalRouting.hpp"
+
+namespace fr {
+
+class FramedRouting {
+ public:
+  FramedRouting() = delete;
+
+  explicit FramedRouting(std::shared_ptr<LocalRouting> localRouting);
+
+  virtual ~FramedRouting() = default;
+
+  [[nodiscard]] uint32_t retrieveUEIp(const uint32_t destination_ip) const;
+
+  [[nodiscard]] uint32_t retrieveUEIp(
+      const pfcp::framed_route_s& framed_route_s) const;
+
+  void addFramedRoute(
+      uint32_t ue_ip, const pfcp::framed_route_s& framed_route_s);
+
+  void remove_entry(uint32_t ue_ip);
+
+  [[nodiscard]] static uint32_t framedIPToUeIP(const std::string& ip) {
+    const char delimeter = '.';
+    uint32_t result      = 0;
+    int shift_counter    = 24;
+    std::string ip_temp;
+    for (auto i = 0; i < ip.length(); ++i) {
+      ip_temp += ip.at(i);
+      if (ip.at(i) == delimeter || i == ip.length() - 1) {
+        result  = result | (std::stoi(ip_temp) << shift_counter);
+        ip_temp = "";
+        shift_counter -= 8;
+      }
+    }
+    return result;
+  };
+
+  [[nodiscard]] static uint32_t frameSubnetToUInt(std::string& subnet) {
+    std::string temp_subnet = "";
+    if (subnet.length() > 2) {
+      return 32;
+    }
+    for (auto i = subnet.length(); i > 0; i--) {
+      temp_subnet.push_back(subnet.at(i - 1));
+    }
+    return std::stoi(temp_subnet);
+  };
+
+  [[nodiscard]] static std::pair<uint32_t, uint32_t> extractIPCidr(
+      const std::string& fr_subnet);
+
+ private:
+  const char pdi_fr_delimeter = ' ';
+
+  std::shared_ptr<LocalRouting> localRouting;
+  std::unordered_map<FramedRoutingKey, uint32_t> KeyToIp{};
+
+  [[nodiscard]] FramedRoutingKey createFramedRoutingKey(
+      std::pair<uint32_t, uint32_t> ipCidr) const;
+
+  [[nodiscard]] RoutingInformation createLocalRoutingInformation(
+      const std::pair<uint32_t, uint32_t>& ipCidr) const;
+
+  [[nodiscard]] SourceNatInformation createLocalSnatInformation(
+      const std::pair<uint32_t, uint32_t>& ipCidr) const;
+};
+
+}  // namespace fr

src/upf_app/framed_routing/FramedRoutingHash.h

+//
+// Created by root on 6/25/24.
+//
+
+#pragma once
+
+// todo(kw) maybe call it subnet or similar
+struct FramedRoutingKey {
+  uint32_t networkAdress{};
+  uint32_t subnet{};
+
+  bool operator==(const FramedRoutingKey& other) const {
+    return (networkAdress == other.networkAdress && subnet == other.subnet);
+  }
+};
+
+template<>
+struct std::hash<FramedRoutingKey> {
+  std::size_t operator()(const FramedRoutingKey& k) const {
+    std::size_t hash = 17;
+    hash *= std::hash<uint32_t>()(k.networkAdress);
+    return hash ^ std::hash<uint32_t>()(k.subnet);
+  }
+};
\ No newline at end of file

src/upf_app/framed_routing/LocalRouting.cpp

+//
+// Created by root on 7/22/24.
+//
+
+#include <fmt/format.h>
+#include <arpa/inet.h>
+#include <iostream>
+#include <logger.hpp>
+#include "LocalRouting.hpp"
+
+namespace fr {
+
+void LocalRouting::add_route(const RoutingInformation& routing_information) {
+  auto original_cerr_streambuf = std::cerr.rdbuf(nullptr);
+  auto cmd                     = fmt::format(
+      "ip route add {}/{} via {} dev tun0", routing_information.destination,
+      routing_information.netmask, routing_information.gateway_address);
+  auto rc = system((const char*) cmd.c_str());
+  if (rc == 0) {
+    Logger::pfcp_switch().info("Route created");
+    this->routeInfoToRtEntry.insert(
+        {routing_information.destination, routing_information});
+  } else {
+    Logger::pfcp_switch().warn("Route information not correct or does exists!");
+  }
+}
+
+void LocalRouting::delete_route(const uint32_t& network_address) {
+  // todo(phine.tech) create function remove dublicated_code
+  auto original_cerr_streambuf = std::cerr.rdbuf(nullptr);
+  std::string destination      = convert_s_addr_to_string(network_address);
+  auto routing_info_iterator   = this->routeInfoToRtEntry.find(destination);
+  if (routing_info_iterator != routeInfoToRtEntry.end()) {
+    const auto routing_information = routing_info_iterator->second;
+    auto cmd                       = fmt::format(
+        "ip route del {}/{} via {} dev tun0", routing_information.destination,
+        routing_information.netmask, routing_information.gateway_address);
+    const auto rc = system((const char*) cmd.c_str());
+    if (rc == 0) {
+      Logger::pfcp_switch().info("Route deleted");
+    } else {
+      Logger::pfcp_switch().warn(
+          "Route information not correct or does not exists!");
+    }
+  }
+};
+
+void LocalRouting::add_source_snat(
+    const SourceNatInformation& source_nat_information) {
+  const std::string cmd = fmt::format(
+      "iptables -t nat -A POSTROUTING -s {}/{} -o {} -j SNAT --to "
+      "{}",
+      source_nat_information.destination, source_nat_information.netmask,
+      source_nat_information.device, source_nat_information.snat_address);
+  const auto rc = system((const char*) cmd.c_str());
+  if (rc == 0) {
+    Logger::pfcp_switch().info("Source NAT added");
+  } else {
+    Logger::pfcp_switch().warn("Source NAT not correct or does exists!");
+  }
+}
+
+void LocalRouting::delete_ssnat(const uint32_t& network_address) {}
+
+std::string LocalRouting::convert_s_addr_to_string(
+    const uint32_t& network_address) {
+  struct in_addr addr;
+  addr.s_addr = htonl(network_address);
+  return inet_ntoa(addr);
+}
+}  // namespace fr
\ No newline at end of file

src/upf_app/framed_routing/LocalRouting.hpp

+//
+// Created by root on 7/22/24.
+//
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+namespace fr {
+struct RoutingInformation {
+  std::string destination;
+  uint32_t netmask = 0;
+  std::string device;
+  std::string gateway_address;
+};
+
+struct SourceNatInformation {
+  std::string destination;
+  uint32_t netmask = 0;
+  std::string device;
+  std::string snat_address;
+};
+// todo (kw) rename class
+class LocalRouting {
+ public:
+  virtual void add_route(const RoutingInformation& routing_information);
+
+  virtual void delete_route(const uint32_t& network_address);
+
+  virtual void add_source_snat(
+      const SourceNatInformation& source_nat_information);
+
+  // todo(phine.tech) implement delete if needed
+  virtual void delete_ssnat(const uint32_t& network_address);
+
+ private:
+  std::unordered_map<std::string, RoutingInformation> routeInfoToRtEntry{};
+
+  std::string convert_s_addr_to_string(const uint32_t& network_address);
+};
+
+}  // namespace fr

src/upf_app/programs/pfcp_session_lookup_xdp_user.cpp

@@ -105,6 +105,35 @@ std::shared_ptr<BPFMap> PFCP_Session_LookupProgram::getSessionMappingMap()
   return mpSessionMappingMap;
 }
 
+/*---------------------------------------------------------------------------------------------------------------*/
+std::shared_ptr<BPFMap> PFCP_Session_LookupProgram::getFramedRouteMappingMap() {
+  return mpFramedRouteMappingMap;
+}
+
+/*---------------------------------------------------------------------------------------------------------------*/
+void PFCP_Session_LookupProgram::updateFramedRouteMappingMap(
+    uint32_t ue_ip, FramedRoutingKeyBPF key) {
+  uint32_t hash_key = hash_framed_routing_key(&key);
+  Logger::upf_app().debug(
+      "Update framed routing map with key: %u, value: %u", hash_key, ue_ip);
+  mpFramedRouteMappingMap->update(hash_key, ue_ip, BPF_ANY);
+}
+
+/*---------------------------------------------------------------------------------------------------------------*/
+void PFCP_Session_LookupProgram::removeFramedRoute(FramedRoutingKeyBPF key) {
+  uint32_t hash_key = hash_framed_routing_key(&key);
+  uint32_t ueip;
+  if (mpFramedRouteMappingMap->lookup(hash_key, &ueip) == 0) {
+    mpFramedRouteMappingMap->remove(hash_key);
+  }
+}
+
+void PFCP_Session_LookupProgram::setFramedRouting(bool enable) {
+  uint8_t value = (enable) ? 1 : 0;
+  uint8_t key   = 0;
+  mpFramedRouteFlagMap->update(key, value, BPF_ANY);
+}
+
 /*---------------------------------------------------------------------------------------------------------------*/
 void PFCP_Session_LookupProgram::initializeMaps() {
   // Store all maps available in the program.
@@ -119,6 +148,10 @@ void PFCP_Session_LookupProgram::initializeMaps() {
       std::make_shared<BPFMap>(mpMaps->getMap("m_next_rule_prog_index"));
   mpSessionMappingMap =
       std::make_shared<BPFMap>(mpMaps->getMap("m_session_mapping"));
+  mpFramedRouteMappingMap =
+      std::make_shared<BPFMap>(mpMaps->getMap("m_framed_route_mapping"));
+  mpFramedRouteFlagMap =
+      std::make_shared<BPFMap>(mpMaps->getMap("framed_routing_flag"));
 }
 
 /*---------------------------------------------------------------------------------------------------------------*/

src/upf_app/programs/pfcp_session_lookup_xdp_user.h

@@ -9,7 +9,7 @@
 #include <signal.h>  // signals
 #include <pfcp_session_lookup_xdp_kernel_skel.h>
 #include <wrappers/BPFMap.hpp>
-// #include "qfi_flow_mapping_table.h"
+#include <framed_routing_bpf.h>
 
 class BPFMaps;
 class BPFMap;
@@ -141,6 +141,14 @@ class PFCP_Session_LookupProgram {
 
   /*---------------------------------------------------------------------------------------------------------------*/
 
+  std::shared_ptr<BPFMap> getFramedRouteMappingMap();
+
+  void updateFramedRouteMappingMap(uint32_t ue_ip, FramedRoutingKeyBPF key);
+
+  void removeFramedRoute(FramedRoutingKeyBPF key);
+
+  void setFramedRouting(bool enable);
+
  private:
   /**
    * @brief Initialize BPF wrappers maps.
@@ -206,6 +214,11 @@ class PFCP_Session_LookupProgram {
   // The UDP interface.
   std::string mUDPInterface;
   /*---------------------------------------------------------------------------------------------------------------*/
+
+  /*---------------------------------------------------------------------------------------------------------------*/
+  // Framed route ue_ip mapping map.
+  std::shared_ptr<BPFMap> mpFramedRouteMappingMap;
+  std::shared_ptr<BPFMap> mpFramedRouteFlagMap;
 };
 
 #endif  // __PFCP_SESSION_LOOKUP_XDP_USER_H__

src/upf_app/simpleswitch/CMakeLists.txt

@@ -39,8 +39,11 @@ include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/pfcp)
 include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/utils)
 include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/utils/bstr)
 include_directories(${SRC_TOP_DIR}/${MOUNTED_COMMON}/model/common_model)
+include_directories(${SRC_TOP_DIR}/upf_app/framed_routing)
 include(${SRC_TOP_DIR}/${MOUNTED_COMMON}/model/nrf/nrf_model.cmake)
 
+
+
 #find_library(FOLLY folly)
 
 add_library (SIMPLE_SWITCH STATIC
@@ -50,5 +53,7 @@ add_library (SIMPLE_SWITCH STATIC
   pfcp_session.cpp
   pfcp_switch.cpp
   simple_switch.cpp
-  )
+  ${SRC_TOP_DIR}/upf_app/framed_routing/FramedRouting.cpp
+  ${SRC_TOP_DIR}/upf_app/framed_routing/LocalRouting.cpp
+)
   

src/upf_app/simpleswitch/pfcp_pdr.cpp

@@ -29,11 +29,13 @@
 #include "pfcp_pdr.hpp"
 #include "upf_n4.hpp"
 #include "logger.hpp"
+#include "upf_config.hpp"
 
 using namespace pfcp;
 using namespace oai::upf::app;
 
 extern upf_n4* upf_n4_inst;
+extern oai::config::upf_config upf_cfg;
 
 //------------------------------------------------------------------------------
 bool pfcp_pdr::look_up_pack_in_access(
@@ -102,7 +104,8 @@ bool pfcp_pdr::look_up_pack_in_core(
       // cause ue_ip_address not present ", pdr_id.rule_id);
       return false;
     }
-    if (pdi.second.ue_ip_address.second.ipv4_address.s_addr != iph->daddr) {
+    if (!upf_cfg.enable_fr &&
+        pdi.second.ue_ip_address.second.ipv4_address.s_addr != iph->daddr) {
       // Logger::pfcp_switch().info( "look_up_pack_in_core failed PDR id %4x,
       // cause PDR ue_ip_address %8X do not match IP dest %8X of packet ",
       //    pdr_id.rule_id, pdi.second.ue_ip_address.second.ipv4_address.s_addr,

src/upf_app/simpleswitch/pfcp_pdr.hpp

@@ -33,7 +33,8 @@
 #include "endpoint.hpp"
 #include "msg_pfcp.hpp"
 #include <mutex>
-
+#include <memory>
+#include "framed_routing/FramedRouting.hpp"
 namespace pfcp {
 
 class pfcp_session;
@@ -50,7 +51,6 @@ class pfcp_pdr {
   std::pair<bool, pfcp::urr_id_t> urr_id;
   std::pair<bool, pfcp::qer_id_t> qer_id;
   std::pair<bool, pfcp::activate_predefined_rules_t> activate_predefined_rules;
-
   bool notified_cp;
 
   explicit pfcp_pdr(uint64_t lseid)

src/upf_app/simpleswitch/pfcp_session.hpp

@@ -33,6 +33,7 @@
 #include "pfcp_far.hpp"
 #include "pfcp_pdr.hpp"
 #include "pfcp_qer.hpp"
+#include "framed_routing/FramedRouting.hpp"
 
 namespace pfcp {
 

src/upf_app/simpleswitch/pfcp_switch.hpp

@@ -50,6 +50,8 @@
 #include <variant>
 
 #include <pthread.h>
+#include "framed_routing/FramedRouting.hpp"
+#include "framed_routing/LocalRouting.hpp"
 
 namespace oai {
 namespace upf {
@@ -93,6 +95,11 @@ class pfcp_switch {
 #define PFCP_SWITCH_MIN_COMMIT_INTERVAL_MILLISECONDS 50
 
   // switching_data_per_cpu_socket             switching_data[];
+
+  const std::shared_ptr<fr::LocalRouting> local_routing =
+      std::make_shared<fr::LocalRouting>();
+  const std::shared_ptr<fr::FramedRouting> fr =
+      std::make_shared<fr::FramedRouting>(local_routing);
   std::unordered_map<pfcp::fseid_t, std::shared_ptr<pfcp::pfcp_session>>
       cp_fseid2pfcp_sessions;
   folly::AtomicHashMap<uint64_t, std::shared_ptr<pfcp::pfcp_session>>

src/upf_app/wrappers/BPFMap.hpp

@@ -2,6 +2,7 @@
 #define __BPFMAP_H__
 
 #include <bpf/bpf.h>
+#include <cerrno>
 #include <bpf/libbpf.h>
 #include "logger.hpp"
 
@@ -108,7 +109,9 @@ int BPFMap::update(KeyType& key, ValueType& value, int flags) {
 
   if (updateReturn != 0) {
     // FIXME: Maybe Key is not support by fmt.
-    Logger::upf_app().error("The key cannot be updated in map");
+    Logger::upf_app().error(
+        "The key cannot be updated in map: %s (errno: %d, %s)", mName.c_str(),
+        errno, strerror(errno));
     throw std::runtime_error("The BPF map cannot be updated");
   } else {
     Logger::upf_app().debug("The key is updated in the map: %s", mName.c_str());