Merge remote-tracking branch 'origin/ci-update-physim-threshold' into integration_2025_w41 (!3679)

CI: Enable Physim threshold check for tests on Caracal and GH

This MR adds support for configurable timing threshold files in PhySim
tests.

- Introduces a new CMake cache variable PHYSIM_CHECK_FILES to specify
  one or more timing threshold files (semicolon-separated).
- Enables optional enforcement of timing thresholds check in PhySim
  tests without changing existing test definitions.
- Defines timing thresholds for LDPC encoding/decoding in nr_ulsim,
  nr_dlsim and ldpctest on Caracal and GH machines.

ci-scripts/cls_oaicitest.py

@@ -255,7 +255,7 @@ def Deploy_Physim(ctx, HTML, node, workdir, script, options):
 	logging.debug(f'Running physims on server {node} workdir {workdir}')
 	with cls_cmd.getConnection(node) as c:
 		sys_info = c.exec_script("scripts/sys-info.sh", 5)
-		ret = c.exec_script(script, 600, options)
+		ret = c.exec_script(script, 1000, options)
 	logging.debug(f'"{script}" finished with code {ret.returncode}, output:\n{ret.stdout}')
 	HTML.CreateHtmlTestRowQueue('Query system info', 'OK', [sys_info.stdout])
 	with cls_cmd.getConnection(node) as ssh:

ci-scripts/docker/Dockerfile.physim.ubuntu

@@ -42,5 +42,6 @@ WORKDIR /oai-ran/build
 RUN cmake -GNinja -DENABLE_PHYSIM_TESTS=ON \
         -DSANITIZE_UNDEFINED=OFF -DSANITIZE_ADDRESS=OFF \
         -DCMAKE_C_FLAGS=-Werror -DCMAKE_CXX_FLAGS=-Werror \
+        -DPHYSIM_CHECK_FILES="ThresholdsGracehopper.cmake" \
         .. && \
     ninja ldpctest polartest smallblocktest nr_pbchsim nr_dlschsim nr_ulschsim nr_dlsim nr_ulsim nr_pucchsim nr_prachsim nr_psbchsim

ci-scripts/xml_files/gnb_usrp_build.xml

@@ -42,7 +42,7 @@
 		<class>Build_eNB</class>
 		<desc>Build gNB (USRP)</desc>
 		<node>caracal</node>
-		<Build_eNB_args>--gNB -w USRP --ninja -c -P --build-lib ldpc_cuda --build-lib ldpc_aal --cmake-opt -DASN1C_EXEC=/opt/asn1c/bin/asn1c</Build_eNB_args>
+		<Build_eNB_args>--gNB -w USRP --ninja -c -P --build-lib ldpc_cuda --build-lib ldpc_aal --cmake-opt -DASN1C_EXEC=/opt/asn1c/bin/asn1c --cmake-opt -DPHYSIM_CHECK_FILES="ThresholdsOffload.cmake"</Build_eNB_args>
 	</testCase>
 
 	<testCase id="100000">

ci-scripts/xml_files/physim_gracehopper.xml

@@ -21,7 +21,7 @@
 
 -->
 <testCaseList>
-        <htmlTabRef>build-tab</htmlTabRef>
+        <htmlTabRef>physim-tab</htmlTabRef>
         <htmlTabName>PhySim</htmlTabName>
         <htmlTabIcon>wrench</htmlTabIcon>
         <TestCaseRequestedList>
@@ -38,9 +38,9 @@
 
         <testCase id="010201">
                 <class>Build_Deploy_Docker_PhySim</class>
-                <desc>Deploy and Deploy PhySim</desc>
+                <desc>Build and Deploy PhySim</desc>
                 <node>localhost</node>
-                <ctest-opt>-R 5g -E physim.5g.nr_dlsim.mcs.mimo.test9 -j16</ctest-opt> <!-- one nr_dlsim test fails on ARM -->
-        </testCase>
+                <ctest-opt>-R 5g -E "nr_dlsim|nr_ulsim|ldpctest" -j32</ctest-opt> <!-- one nr_dlsim test fails on ARM -->
+	</testCase>
 
 </testCaseList>

ci-scripts/xml_files/physim_timed_gracehopper.xml

+<!--
+
+ Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The OpenAirInterface Software Alliance licenses this file to You under
+ the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.openairinterface.org/?page_id=698
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ For more information about the OpenAirInterface (OAI) Software Alliance:
+      contact@openairinterface.org
+
+-->
+<testCaseList>
+        <htmlTabRef>physim-timed-tab</htmlTabRef>
+        <htmlTabName>PhySim with timing check</htmlTabName>
+        <htmlTabIcon>wrench</htmlTabIcon>
+        <TestCaseRequestedList>
+          800813
+	  010202
+        </TestCaseRequestedList>
+        <TestCaseExclusionList></TestCaseExclusionList>
+
+        <testCase id="800813">
+                <class>Create_Workspace</class>
+                <desc>Create new Workspace</desc>
+                <node>localhost</node>
+        </testCase>
+
+	<testCase id="010202">
+		<class>Build_Deploy_Docker_PhySim</class>
+		<desc>Build and Deploy PhySim with timing check</desc>
+		<node>localhost</node>
+		<ctest-opt>-R 5g -R "nr_dlsim|nr_ulsim|ldpctest" -E physim.5g.nr_dlsim.mcs.mimo.test9 -j1</ctest-opt> <!-- one nr_dlsim test fails on ARM -->
+	</testCase>
+
+</testCaseList>

docker/Dockerfile.phySim.rhel9

@@ -106,4 +106,11 @@ COPY --from=phy-sim-build \
     /oai-ran/cmake_targets/ran_build/build/openair1/SIMULATION/tests/CTestTestfile.cmake \
     /opt/oai-physim/
 
+RUN mkdir -p /oai-ran/openair1/SIMULATION/tests
+
+COPY --from=phy-sim-build \
+    /oai-ran/openair1/SIMULATION/tests/RunTimedTest.cmake \
+    /oai-ran/openair1/SIMULATION/tests/analyze-timing.sh \
+    /oai-ran/openair1/SIMULATION/tests/
+
 #CMD ["sleep", "infinity"]

openair1/SIMULATION/tests/ThresholdsOffload.cmake

-check_threshold(physim.5g-offload.nr_dlsim.testCPU.1 "DLSCH encoding time" "< 230")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.1 "DLSCH encoding time" "< 100")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.2 "DLSCH encoding time" "< 300")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.2 "DLSCH encoding time" "< 100")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.3 "DLSCH encoding time" "< 350")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.3 "DLSCH encoding time" "< 200")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.4 "DLSCH encoding time" "< 300")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.4 "DLSCH encoding time" "< 150")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.5 "DLSCH encoding time" "< 350")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.5 "DLSCH encoding time" "< 250")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.6 "DLSCH encoding time" "< 400")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.6 "DLSCH encoding time" "< 400")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.7 "DLSCH encoding time" "< 400")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.7 "DLSCH encoding time" "< 200")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.8 "DLSCH encoding time" "< 450")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.8 "DLSCH encoding time" "< 500")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.9 "DLSCH encoding time" "< 500")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.9 "DLSCH encoding time" "< 500")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.10 "DLSCH encoding time" "< 400")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.10 "DLSCH encoding time" "< 200")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.11 "DLSCH encoding time" "< 400")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.11 "DLSCH encoding time" "< 300")
-check_threshold(physim.5g-offload.nr_dlsim.testCPU.12 "DLSCH encoding time" "< 450")
-check_threshold(physim.5g-offload.nr_dlsim.testT2.12 "DLSCH encoding time" "< 450")
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.1 "DLSCH encoding time" AVG 50 ABS_VAR 10)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.1 "DLSCH encoding time" AVG 19 ABS_VAR 4)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.2 "DLSCH encoding time" AVG 65 ABS_VAR 13)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.2 "DLSCH encoding time" AVG 27 ABS_VAR 6)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.3 "DLSCH encoding time" AVG 97 ABS_VAR 20)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.3 "DLSCH encoding time" AVG 33 ABS_VAR 7)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.4 "DLSCH encoding time" AVG 68 ABS_VAR 14)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.4 "DLSCH encoding time" AVG 26 ABS_VAR 6)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.5 "DLSCH encoding time" AVG 108 ABS_VAR 22)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.5 "DLSCH encoding time" AVG 46 ABS_VAR 10)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.6 "DLSCH encoding time" AVG 131 ABS_VAR 27)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.6 "DLSCH encoding time" AVG 80 ABS_VAR 16)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.7 "DLSCH encoding time" AVG 120 ABS_VAR 24)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.7 "DLSCH encoding time" AVG 44 ABS_VAR 9)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.8 "DLSCH encoding time" AVG 146 ABS_VAR 30)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.8 "DLSCH encoding time" AVG 87 ABS_VAR 18)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.9 "DLSCH encoding time" AVG 204 ABS_VAR 41)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.9 "DLSCH encoding time" AVG 138 ABS_VAR 28)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.10 "DLSCH encoding time" AVG 149 ABS_VAR 30)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.10 "DLSCH encoding time" AVG 49 ABS_VAR 10)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.11 "DLSCH encoding time" AVG 164 ABS_VAR 33)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.11 "DLSCH encoding time" AVG 90 ABS_VAR 18)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testCPU.12 "DLSCH encoding time" AVG 223 ABS_VAR 45)
+check_threshold_variance(physim.5g-offload.nr_dlsim.testT2.12 "DLSCH encoding time" AVG 146 ABS_VAR 30)
 
-check_threshold(physim.5g-offload.nr_ulsim.testCPU.1 "ULSCH total decoding time" "< 300")
-check_threshold(physim.5g-offload.nr_ulsim.testT2.1 "ULSCH total decoding time" "< 100")
-check_threshold(physim.5g-offload.nr_ulsim.testCPU.2 "ULSCH total decoding time" "< 300")
-check_threshold(physim.5g-offload.nr_ulsim.testT2.2 "ULSCH total decoding time" "< 150")
-check_threshold(physim.5g-offload.nr_ulsim.testCPU.3 "ULSCH total decoding time" "< 250")
-check_threshold(physim.5g-offload.nr_ulsim.testT2.3 "ULSCH total decoding time" "< 250")
-check_threshold(physim.5g-offload.nr_ulsim.testCPU.4 "ULSCH total decoding time" "< 300")
-check_threshold(physim.5g-offload.nr_ulsim.testT2.4 "ULSCH total decoding time" "< 150")
-check_threshold(physim.5g-offload.nr_ulsim.testCPU.5 "ULSCH total decoding time" "< 400")
-check_threshold(physim.5g-offload.nr_ulsim.testT2.5 "ULSCH total decoding time" "< 350")
-check_threshold(physim.5g-offload.nr_ulsim.testCPU.6 "ULSCH total decoding time" "< 400")
-check_threshold(physim.5g-offload.nr_ulsim.testT2.6 "ULSCH total decoding time" "< 550")
-check_threshold(physim.5g-offload.nr_ulsim.testCPU.7 "ULSCH total decoding time" "< 300")
-check_threshold(physim.5g-offload.nr_ulsim.testT2.7 "ULSCH total decoding time" "< 250")
-check_threshold(physim.5g-offload.nr_ulsim.testCPU.8 "ULSCH total decoding time" "< 600")
-check_threshold(physim.5g-offload.nr_ulsim.testT2.8 "ULSCH total decoding time" "< 650")
-check_threshold(physim.5g-offload.nr_ulsim.testCPU.9 "ULSCH total decoding time" "< 650")
-check_threshold(physim.5g-offload.nr_ulsim.testT2.9 "ULSCH total decoding time" "< 1100")
+check_threshold_variance(physim.5g-offload.nr_ulsim.testCPU.1 "ULSCH total decoding time" AVG 141 ABS_VAR 29)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testT2.1 "ULSCH total decoding time" AVG 40 ABS_VAR 8)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testCPU.2 "ULSCH total decoding time" AVG 165 ABS_VAR 33)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testT2.2 "ULSCH total decoding time" AVG 63 ABS_VAR 13)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testCPU.3 "ULSCH total decoding time" AVG 156 ABS_VAR 32)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testT2.3 "ULSCH total decoding time" AVG 87 ABS_VAR 18)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testCPU.4 "ULSCH total decoding time" AVG 170 ABS_VAR 34)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testT2.4 "ULSCH total decoding time" AVG 74 ABS_VAR 15)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testCPU.5 "ULSCH total decoding time" AVG 299 ABS_VAR 60)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testT2.5 "ULSCH total decoding time" AVG 120 ABS_VAR 24)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testCPU.6 "ULSCH total decoding time" AVG 324 ABS_VAR 65)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testT2.6 "ULSCH total decoding time" AVG 180 ABS_VAR 36)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testCPU.7 "ULSCH total decoding time" AVG 200 ABS_VAR 40)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testT2.7 "ULSCH total decoding time" AVG 120 ABS_VAR 24)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testCPU.8 "ULSCH total decoding time" AVG 436 ABS_VAR 88)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testT2.8 "ULSCH total decoding time" AVG 229 ABS_VAR 46)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testCPU.9 "ULSCH total decoding time" AVG 530 ABS_VAR 106)
+check_threshold_variance(physim.5g-offload.nr_ulsim.testT2.9 "ULSCH total decoding time" AVG 334 ABS_VAR 67)

openair1/SIMULATION/tests/analyze-timing.sh

@@ -6,12 +6,9 @@
 
 function die() { echo $@ 1>&2; exit 1; }
 
-# RC will be the return code. If any rule fails, it will set RC=1, which will
-# make the script fail. Print also every line, because the logs are piped into
-# this script, but a user is typically also interested into the raw logs.
+# Print every line, because the logs are piped into this script, but a user is
+# typically also interested into the raw logs.
 SCRIPT='
-  BEGIN { RC = 0; }
-
   { print $0 }
 '
 
@@ -24,9 +21,16 @@ while [ $# -gt 0 ]; do
   shift 2
 
   # Add a rule that searches for a PATTERN + number, and checks against
-  # CONDition. If the condition does not hold, it is counted as a failure (sets
-  # RC to signal error). In both cases, the result is logged in an array to
-  # output at the end of the script.
+  # CONDition. If the condition does not hold, it is counted as a failure. In
+  # both cases, the result is logged in an array to output at the end of the
+  # script.
+  #
+  # To search for the number, first substr() returns what has been matched
+  # (PATTERN + number), and sub() deletes PATTERN and whitespace, resulting in
+  # the number (measurement). To force number into a numeric context, we add 0,
+  # as awk would otherwise compare a string to a number (example:
+  #   gawk 'BEGIN { meas = "117"; if (meas > 96) { } else { print "fail";}}'
+  # printfs "fail").
   #
   # Example: pattern "PHY proc tx", condition "< 200"
   # The awk script tries to match every line for "PHY proc tx _NUMBER_" (where
@@ -35,14 +39,15 @@ while [ $# -gt 0 ]; do
   # If the condition holds, will set "CHECK PHY proc tx _NUMBER_ < 200 SUCCESS".
   # If the condition fails, will set "CHECK PHY proc tx _NUMBER_ < 200 FAIL".
   SCRIPT+='
-  match($0, /'${PATTERN}' +([0-9]+(\.[0-9]+)?)/, n) {
-    if (n[1] '${COND}') {
+  match($0, /'${PATTERN}' +([0-9]+(\.[0-9]+)?)/) {
+    meas = substr($0, RSTART, RLENGTH);
+    sub(/'${PATTERN}' +/, "", meas);
+    if ((meas+0) '${COND}') {
       r = "SUCCESS";
     } else {
       r = "FAIL";
-      RC = 1;
     }
-    RESULTS['${NUM}']=sprintf("CHECK %-35s %7.2f %-8s %s", "'${PATTERN}'", n[1], " '${COND}'", r);
+    RESULTS['${NUM}']=sprintf("CHECK %-35s %7.2f %-8s %s", "'${PATTERN}'", meas, " '${COND}'", r);
   }
 '
 
@@ -54,7 +59,6 @@ while [ $# -gt 0 ]; do
   END {
     if (!RESULTS['${NUM}']) {
       RESULTS['${NUM}']=sprintf("CHECK %-35s          %-7s NOTFOUND", "'${PATTERN}'", "'${COND}'");
-      RC = 1;
     }
   }
 '
@@ -66,8 +70,11 @@ done
 # (0 on success, i.e..all conditions checked, otherwise 1 on failure).
 SCRIPT+='
   END {
+    RC = 0;
     for (i = 0; i < '${NUM}'; ++i) {
       print RESULTS[i]
+      if (!(RESULTS[i] ~ /SUCCESS$/))
+        RC = 1;
     }
     exit RC
   }