LTTng logger (CTF)

This commit introduces LTTng logging mechanism which is an open source
tracing tool.

URL: https://lttng.org/

CMakeLists.txt

@@ -284,6 +284,7 @@ add_boolean_option(DEBUG_ASN1 False "Enable ASN1 debug logs" OFF)
 # see common/utils/config.h
 add_boolean_option(TRACE_ASN1C_ENC_DEC OFF "Enable ASN1 encoder/decoder debug traces via OAI logging system" ON)
 add_boolean_option(T_TRACER            True  "Activate the T tracer, a debugging/monitoring framework" ON)
+add_boolean_option(ENABLE_LTTNG        False "Activate the LTTNG tracer, a debugging/monitoring framework" ON)
 add_boolean_option(UE_AUTOTEST_TRACE   False "Activate UE autotest specific logs" ON)
 add_boolean_option(UE_DEBUG_TRACE      False "Activate UE debug trace" ON)
 add_boolean_option(UE_TIMING_TRACE     False "Activate UE timing trace" ON)
@@ -659,6 +660,14 @@ add_library(UTIL
   ${OPENAIR_DIR}/common/utils/time_meas.c
   ${OPENAIR_DIR}/common/utils/time_stat.c
 )
+if (ENABLE_LTTNG)
+  find_package(LTTngUST 2.3.8 EXACT REQUIRED)
+  target_sources(UTIL PRIVATE common/utils/LOG/lttng-tp.c)
+  target_link_libraries(UTIL PUBLIC lttng-ust)
+else()
+  message(STATUS "LTTNG support disabled")
+endif()
+
 pkg_check_modules(cap libcap)
 if (cap_FOUND)
   # see system.c for more info

cmake_targets/build_oai

@@ -124,6 +124,8 @@ Options:
 --disable-cpu-affinity
    Disables CPU Affinity between UHD/TX/RX Threads (Valid only when deadline scheduler is disabled). By defaulT, CPU Affinity is enabled when not using deadline scheduler. It is enabled only with >2 CPUs. For eNB, CPU_0-> Device library (UHD), CPU_1->TX Threads, CPU_2...CPU_MAX->Rx Threads. For UE, CPU_0->Device Library(UHD), CPU_1..CPU_MAX -> All the UE threads
 --enable-cpu-affinity
+--enable-LTTNG
+   Enable LTTNG log. While lttng is enabled, T-Tracer will be disabled
 --disable-T-Tracer
    Disables the T tracer.
 --disable-hardware-dependency
@@ -353,6 +355,11 @@ function main() {
            BUILD_DOXYGEN=1
            echo_info "Will build doxygen support"
            shift;;     
+       --enable-LTTNG)
+            CMAKE_CMD="$CMAKE_CMD -DENABLE_LTTNG=ON -DT_TRACER=OFF"
+            echo_info "Enable LTTNG tracer"
+            USE_LTTNG=1
+            shift 1;;
        --disable-T-Tracer)
             CMAKE_CMD="$CMAKE_CMD -DT_TRACER=False"
             echo_info "Disabling the T tracer"
@@ -454,6 +461,10 @@ function main() {
   if [ "$INSTALL_EXTERNAL" = "1" ] ; then
     echo_info "Installing packages"
     check_install_oai_software
+    if [ "$USE_LTTNG" = "1" ] ; then
+      echo_info "installing LTTng"
+      install_lttng $1
+    fi
     if [ "$HW" == "OAI_USRP" ] ; then
       echo_info "installing packages for USRP support"
       check_install_usrp_uhd_driver

cmake_targets/tools/build_helper

@@ -704,6 +704,97 @@ install_simde_from_source(){
     $SUDO \cp -rv ../simde /usr/include
 }
 
+install_lttng() {
+    if [[ "$OS_DISTRO" != "ubuntu" ]]; then
+      echo_info "LTTng is supported only for ubuntu at the moment."
+      echo_info $distribution
+      return
+    fi
+
+    set +e
+    $SUDO apt-add-repository ppa:lttng/stable-2.13 -y && apt-get update
+    $SUDO apt-get install -y lttng-tools lttng-modules-dkms liblttng-ust-dev
+    if [[ $? > 0 ]]; then
+      echo "Failed to install LTTng using apt. Installing from source"
+      install_lttng_deps
+      install_lttng_from_source
+    fi
+    set -e
+}
+
+install_lttng_deps() {
+  echo "installing dependencies of LTTng"
+  $SUDO apt-get install uuid-dev libxml2-dev -y
+}
+
+install_lttng_from_source() {
+    echo "installing LTTng from source"
+    install_popt_1_18_from_source
+    install_userspace_rcu
+    install_lttng_ust_2_13_from_source
+    install_lttng_tools_2_13_from_source
+}
+
+install_popt_1_18_from_source() {
+  $SUDO $INSTALLER -y install wget
+  echo_info "\nInstalling popt_1_18."
+  cd $(mktemp -d) &&
+  wget http://ftp.rpm.org/popt/releases/popt-1.x/popt-1.18.tar.gz
+  tar -xf popt-1.18.tar.gz && cd popt-1.18
+  ./configure && make -j16 && $SUDO make install
+}
+
+install_userspace_rcu() {
+    echo_info "\nInstalling U-RCU from source."
+    cd /tmp
+    $SUDO rm -rf /tmp/urcu
+    git clone https://github.com/urcu/userspace-rcu.git /tmp/urcu
+    cd /tmp/urcu
+    git checkout stable-0.12
+    ./bootstrap && ./configure && make
+    $SUDO make install
+    $SUDO ldconfig
+}
+
+
+install_lttng_modules_2_13_from_source() {
+    $SUDO $INSTALLER -y install wget
+    echo_info "\nInstalling LTTng Modules."
+    cd $(mktemp -d) &&
+    wget https://lttng.org/files/lttng-modules/lttng-modules-latest-2.13.tar.bz2 &&
+    tar -xf lttng-modules-latest-2.13.tar.bz2 &&
+    cd lttng-modules-2.13.* &&
+    make &&
+    $SUDO make modules_install &&
+    $SUDO depmod -a
+}
+
+install_lttng_ust_2_13_from_source() {
+    $SUDO $INSTALLER -y install wget
+    echo_info "\nInstalling LTTng UST."
+    cd $(mktemp -d) &&
+    wget https://lttng.org/files/lttng-ust/lttng-ust-latest-2.13.tar.bz2 &&
+    tar -xf lttng-ust-latest-2.13.tar.bz2 &&
+    cd lttng-ust-2.13.* &&
+    ./configure --disable-numa &&
+    make &&
+    $SUDO make install &&
+    $SUDO ldconfig
+}
+
+install_lttng_tools_2_13_from_source() {
+    $SUDO $INSTALLER -y install wget
+    echo_info "\nInstalling LTTng Tools."
+    cd $(mktemp -d) &&
+    wget https://lttng.org/files/lttng-tools/lttng-tools-latest-2.13.tar.bz2 &&
+    tar -xf lttng-tools-latest-2.13.tar.bz2 &&
+    cd lttng-tools-2.13.* &&
+    ./configure &&
+    make &&
+    $SUDO make install &&
+    $SUDO ldconfig
+}
+
 #################################################
 # 2. compile
 ################################################

common/utils/LOG/DOC/log.md

@@ -15,5 +15,6 @@ By default, this facility is included at build-time and activated at run-time. T
 * [runtime usage](rtusage.md)
 * [developer usage](devusage.md)
 * [module architecture](arch.md)
+* [lttng usage](lttng_logs.md)
 
 [oai Wikis home](https://gitlab.eurecom.fr/oai/openairinterface5g/wikis/home)

common/utils/LOG/DOC/lttng_logs.md

+# OAI gNB LTTng Tracing Setup Guide
+
+## Overview
+
+This guide will walk you through setting up tracing for an OpenAirInterface (OAI) gNB (gNodeB) using LTTng (Linux Trace Toolkit Next Generation) and Babeltrace.
+### What is LTTng and Why Use It?
+
+LTTng, or Linux Trace Toolkit Next Generation, is a powerful logging framework designed for Linux systems. It provides low-overhead tracing capabilities, allowing developers to monitor and analyze system behavior in real-time without significant performance impact. LTTng offers several advantages:
+
+- **Low Overhead**: LTTng introduces minimal overhead to the system, making it suitable for use in production environments without affecting system performance.
+
+- **Customizable**: LTTng allows users to define custom tracepoints in their applications, providing fine-grained control over what events to trace and collect.
+
+- **Scalability**: It can scale to large distributed systems, making it suitable for tracing complex software stacks across multiple nodes.
+
+## Prerequisites
+
+- Ubuntu system
+
+Note: only LTTng 2.3.8 is supported.
+
+## Building OAI gNB
+
+1. **Clean and Build OAI gNB with LTTng:**
+
+    1.1  Install Dependencies
+
+    ```bash
+    ./build_oai --ninja -I --clean --enable-LTTNG
+    
+    ```
+    1.2  Build gNB and nrUE
+    ```
+    ./build_oai --ninja --gNB --nrUE -w SIMU --enable-LTTNG
+    ```
+## Setting up LTTng
+
+1. **Start LTTng Session and Relay:**
+
+    ```bash
+    sudo lttng-sessiond -d
+    sudo lttng-relayd -d
+    ```
+
+2. **Create Live LTTng Session:**
+
+    ```bash
+    sudo lttng create my-session --live --set-url=net://127.0.0.1
+    ```
+
+3. **Enable gNB Trace Events:**
+
+    ```bash
+    sudo lttng enable-event --userspace OAI:gNB
+    ```
+
+4. **Start LTTng Tracing:**
+
+    ```bash
+    sudo lttng start
+    ```
+
+## Running the gNB
+
+1. **Run gNB:**
+
+    ```bash
+    ./$binary_path -O $configuration_file PARALLEL_SINGLE_THREAD --rfsimulator.serveraddr server --rfsim --sa -E
+    ```
+
+## Verifying Tracepoints
+
+1. **List Active Tracepoints:**
+
+    ```bash
+    sudo lttng list -u
+    ```
+
+    *Possible Output:*
+
+    ```
+    UST events:
+    -------------
+    
+    PID: 1154722 - Name: /home/firecell/Desktop/FirecellRepos/firecellrd-oai5g-ran/cmake_targets/ran_build/build/nr-softmodem
+          OAI:gNB (loglevel: TRACE_DEBUG_FUNCTION (12)) (type: tracepoint)
+    ```
+
+## Analyzing Traces
+
+1. **Install Babeltrace:**
+
+    ```bash
+    sudo apt-get install babeltrace
+    ```
+
+
+2. **Capture Trace Logs Live:**
+
+    - To learn the full path of the trace session:
+    
+        ```bash
+        babeltrace --input-format=lttng-live net://localhost
+        ```
+
+    - Trace logs using the full path:
+    
+        ```bash
+        babeltrace --input-format=lttng-live net://localhost/host/firecell-XPS-15-9530/my-session
+        ```
+
+    *Possible Output:*
+
+    ```
+    [19:35:32.181608002] (+2.664882127) firecell-XPS-15-9530 OAI:gNB: { cpu_id = 10 }, { MODNAME = "OAI-NR_MAC info", EVENTID = -1, SFN = -1, SLOT = -1, FUNCTION = "gNB_dlsch_ulsch_scheduler", LINE = 246, MSG = "Frame.Slot 0.0\n" }
+    ```
+3. **Capture Trace Logs Offline:**
+
+    - Create an offline trace session with a specified output directory:
+
+        ```bash
+        sudo lttng create offline_session --output=/home/trace_offline/
+        ```
+
+    - Enable gNB trace events:
+
+        ```bash
+        sudo lttng enable-event --userspace OAI:gNB
+        ```
+
+    - Start capturing trace logs:
+
+        ```bash
+        sudo lttng start
+        ```
+
+    - Stop the trace capture:
+
+        ```bash
+        sudo lttng stop
+        ```
+
+    - Destroy the trace session:
+
+        ```bash
+        sudo lttng destroy
+        ```
+
+    - Use Babeltrace to analyze the captured trace logs:
+
+        ```bash
+        sudo babeltrace /home/trace_offline/
+        ```

common/utils/LOG/log.c

@@ -603,6 +603,25 @@ void logRecord_mt(const char *file,
   log_output_memory(c, file,func,line,comp,level,format,args);
   va_end(args);
 }
+#if ENABLE_LTTNG
+void logRecord_lttng(const char *file, const char *func, int line, int comp, int level, const char *format, ...)
+{
+  log_component_t *c = &g_log->log_component[comp];
+  char header[48];
+  char buf[MAX_LOG_TOTAL];
+  va_list args;
+  va_start(args, format);
+  vsnprintf(buf, sizeof(buf) - 1, format, args);
+  va_end(args);
+
+  if (map_int_to_str(log_level_names, level) != NULL)
+    snprintf(header, sizeof(header), "OAI-%s %s", c->name, map_int_to_str(log_level_names, level));
+  else
+    snprintf(header, sizeof(header), "OAI-%s", c->name);
+
+  LOG_FC(header, func, line, buf);
+}
+#endif
 
 void vlogRecord_mt(const char *file,
 		   const char *func,

common/utils/LOG/log.h

@@ -52,6 +52,9 @@
 #endif
 #include <pthread.h>
 #include <common/utils/utils.h>
+#if ENABLE_LTTNG
+#include "lttng-log.h"
+#endif
 /*----------------------------------------------------------------------------*/
 #include <assert.h>
 #ifdef NDEBUG
@@ -234,6 +237,10 @@ void logTerm (void);
 int  isLogInitDone (void);
 void logRecord_mt(const char *file, const char *func, int line,int comp, int level, const char *format, ...) __attribute__ ((format (printf, 6, 7)));
 void vlogRecord_mt(const char *file, const char *func, int line, int comp, int level, const char *format, va_list args );
+#if ENABLE_LTTNG
+void logRecord_lttng(const char *file, const char *func, int line, int comp, int level, const char *format, ...)
+    __attribute__((format(printf, 6, 7)));
+#endif
 void log_dump(int component, void *buffer, int buffsize,int datatype, const char *format, ... );
 int  set_log(int component, int level);
 void set_glog(int level);
@@ -425,9 +432,50 @@ int32_t write_file_matlab(const char *fname, const char *vname, void *data, int
 
 /* define variable only used in LOG macro's */
 #define LOG_VAR(A, B) A B
-
-#else /* no T_TRACER */
-
+#else /* T_TRACER */
+#if ENABLE_LTTNG
+#define LOG_E(c, x...)                                                     \
+  do {                                                                     \
+    if (g_log->log_component[c].level >= OAILOG_ERR) {                     \
+      logRecord_lttng(__FILE__, __FUNCTION__, __LINE__, c, OAILOG_ERR, x); \
+    }                                                                      \
+  } while (0)
+#define LOG_W(c, x...)                                                         \
+  do {                                                                         \
+    if (g_log->log_component[c].level >= OAILOG_WARNING) {                     \
+      logRecord_lttng(__FILE__, __FUNCTION__, __LINE__, c, OAILOG_WARNING, x); \
+    }                                                                          \
+  } while (0)
+#define LOG_A(c, x...)                                                          \
+  do {                                                                          \
+    if (g_log->log_component[c].level >= OAILOG_ANALYSIS) {                     \
+      logRecord_lttng(__FILE__, __FUNCTION__, __LINE__, c, OAILOG_ANALYSIS, x); \
+    }                                                                           \
+  } while (0)
+#define LOG_I(c, x...)                                                      \
+  do {                                                                      \
+    if (g_log->log_component[c].level >= OAILOG_INFO) {                     \
+      logRecord_lttng(__FILE__, __FUNCTION__, __LINE__, c, OAILOG_INFO, x); \
+    }                                                                       \
+  } while (0)
+#define LOG_D(c, x...)                                                       \
+  do {                                                                       \
+    if (g_log->log_component[c].level >= OAILOG_DEBUG) {                     \
+      logRecord_lttng(__FILE__, __FUNCTION__, __LINE__, c, OAILOG_DEBUG, x); \
+    }                                                                        \
+  } while (0)
+#define LOG_T(c, x...)                                                       \
+  do {                                                                       \
+    if (g_log->log_component[c].level >= OAILOG_TRACE) {                     \
+      logRecord_lttng(__FILE__, __FUNCTION__, __LINE__, c, OAILOG_TRACE, x); \
+    }                                                                        \
+  } while (0)
+#define LOG_DDUMP(c, b, s, f, x...)                    \
+  do {                                                 \
+    if (g_log->log_component[c].level >= OAILOG_DEBUG) \
+      log_dump(c, b, s, f, x);                         \
+  } while (0)
+#else
 #define LOG_E(c, x...)                                                  \
   do {                                                                  \
     if (g_log->log_component[c].level >= OAILOG_ERR)                    \
@@ -469,6 +517,7 @@ int32_t write_file_matlab(const char *fname, const char *vname, void *data, int
     if (g_log->log_component[c].level >= OAILOG_TRACE)                    \
       logRecord_mt(__FILE__, __FUNCTION__, __LINE__, c, OAILOG_TRACE, x); \
   } while (0)
+#endif
 
 #define VLOG(c, l, f, args)                                           \
   do {                                                                \

common/utils/LOG/lttng-log.h

+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*! \file lttng-log.h
+* \brief LTTng Log interface
+* \author Anurag Asokan
+* \date 2024
+* \version 0.5
+* @ingroup util
+
+*/
+
+#ifndef __LTTNG_LOG_H__
+#define __LTTNG_LOG_H__
+#if ENABLE_LTTNG
+#include "lttng-tp.h"
+
+#define LOG_FC(component, func, line, log)                        \
+  do {                                                            \
+    tracepoint(OAI, gNB, component, -1, -1, -1, func, line, log); \
+  } while (0)
+#else
+#define LOG_FC(component, func, line, log)
+#endif
+
+#endif /** __LTTNG_LOG_H__ */

common/utils/LOG/lttng-tp.c

+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*! \file lttng-tp.c
+* \brief Lttng trace implementaion
+* \author Anurag Asokan
+* \date 2024
+* \version 0.5
+* @ingroup util
+
+*/
+//This file is used to define the tracepoints for the lttng tracing.In order to to do linkage with the lttng-tp.h file, we need to define the tracepoints here.
+#define TRACEPOINT_CREATE_PROBES
+#define TRACEPOINT_DEFINE
+
+#include "lttng-tp.h"

common/utils/LOG/lttng-tp.h

+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*! \file lttng-tp.h
+* \brief Lttng tracer implementaion
+* \author Anurag Asokan
+* \date 2024
+* \version 0.5
+* @ingroup util
+
+*/
+
+#ifndef _LTTNG_TP_H
+#define _LTTNG_TP_H
+
+#undef TRACEPOINT_PROVIDER
+#define TRACEPOINT_PROVIDER OAI
+
+#undef TRACEPOINT_INCLUDE
+#define TRACEPOINT_INCLUDE "./common/utils/LOG/lttng-tp.h"
+
+#include <stdbool.h>
+#include <lttng/tracepoint.h>
+
+TRACEPOINT_EVENT(
+    OAI,
+    gNB,
+    TP_ARGS(const char*, log_modName, int, event_id, int, sfn, int, slot, const char*, funcName, int, lineNo, const char*, msg),
+    TP_FIELDS(ctf_string(MODNAME, log_modName) ctf_integer(int32_t, EVENTID, event_id) ctf_integer(int32_t, SFN, sfn)
+                  ctf_integer(int32_t, SLOT, slot) ctf_string(FUNCTION, funcName) ctf_integer(int32_t, LINE, lineNo)
+                      ctf_string(MSG, msg)))
+TRACEPOINT_LOGLEVEL(OAI, gNB, TRACE_DEBUG_FUNCTION)
+
+#endif /* _LTTNG_TP_H */
+
+#include <lttng/tracepoint-event.h>