lcore: not check socket id if MTL_FLAG_BIND_NUMA not enabled (#864)

./build/app/RxTxApp --config_file tests/script/loop_json/1080p59_1v.json --not_bind_numa Signed-off-by: Frank Du <[email protected]>
OpenVisualCloud · May 16, 2024 · 5e971e6 · 5e971e6
1 parent f51b820
commit 5e971e6
Show file tree

Hide file tree

Showing 16 changed files with 63 additions and 5 deletions.
diff --git a/app/src/app_base.h b/app/src/app_base.h
@@ -626,7 +626,8 @@ struct st_app_context {
   int rx_video_fb_cnt;
   int rx_video_rtp_ring_size; /* the ring size for rx video rtp type */
   bool has_sdl;               /* has SDL device or not*/
-  int rx_audio_dump_time_s;   /* the audio dump time */
+  bool rx_video_multi_thread;
+  int rx_audio_dump_time_s; /* the audio dump time */
 
   struct st_app_rx_audio_session* rx_audio_sessions;
   int rx_audio_session_cnt;

diff --git a/app/src/args.c b/app/src/args.c
@@ -43,6 +43,7 @@ enum st_args_cmd {
   ST_ARG_RX_VIDEO_FILE_FRAMES,
   ST_ARG_RX_VIDEO_FB_CNT,
   ST_ARG_RX_VIDEO_RTP_RING_SIZE,
+  ST_ARG_RX_VIDEO_MULTI_THREADS,
   ST_ARG_RX_AUDIO_SESSIONS_CNT,
   ST_ARG_RX_AUDIO_RTP_RING_SIZE,
   ST_ARG_RX_AUDIO_DUMP_TIME_S,
@@ -61,6 +62,8 @@ enum st_args_cmd {
   ST_ARG_TX_DISPLAY,
   ST_ARG_RX_DISPLAY,
   ST_ARG_DISABLE_MIGRATE,
+  ST_ARG_BIND_NUMA,
+  ST_ARG_NOT_BIND_NUMA,
 
   ST_ARG_CONFIG_FILE = 0x300,
   ST_ARG_TEST_TIME,
@@ -181,6 +184,7 @@ static struct option st_app_args_options[] = {
     {"rx_video_file_frames", required_argument, 0, ST_ARG_RX_VIDEO_FILE_FRAMES},
     {"rx_video_fb_cnt", required_argument, 0, ST_ARG_RX_VIDEO_FB_CNT},
     {"rx_video_rtp_ring_size", required_argument, 0, ST_ARG_RX_VIDEO_RTP_RING_SIZE},
+    {"rx_video_multi_thread", no_argument, 0, ST_ARG_RX_VIDEO_MULTI_THREADS},
     {"rx_audio_sessions_count", required_argument, 0, ST_ARG_RX_AUDIO_SESSIONS_CNT},
     {"rx_audio_rtp_ring_size", required_argument, 0, ST_ARG_RX_AUDIO_RTP_RING_SIZE},
     {"rx_audio_dump_time_s", required_argument, 0, ST_ARG_RX_AUDIO_DUMP_TIME_S},
@@ -198,6 +202,8 @@ static struct option st_app_args_options[] = {
     {"tx_display", no_argument, 0, ST_ARG_TX_DISPLAY},
     {"rx_display", no_argument, 0, ST_ARG_RX_DISPLAY},
     {"disable_migrate", no_argument, 0, ST_ARG_DISABLE_MIGRATE},
+    {"bind_numa", no_argument, 0, ST_ARG_BIND_NUMA},
+    {"not_bind_numa", no_argument, 0, ST_ARG_NOT_BIND_NUMA},
 
     {"config_file", required_argument, 0, ST_ARG_CONFIG_FILE},
     {"test_time", required_argument, 0, ST_ARG_TEST_TIME},
@@ -507,6 +513,9 @@ int st_app_parse_args(struct st_app_context* ctx, struct mtl_init_params* p, int
       case ST_ARG_RX_VIDEO_RTP_RING_SIZE:
         ctx->rx_video_rtp_ring_size = atoi(optarg);
         break;
+      case ST_ARG_RX_VIDEO_MULTI_THREADS:
+        ctx->rx_video_multi_thread = true;
+        break;
       case ST_ARG_RX_AUDIO_SESSIONS_CNT:
         ctx->rx_audio_session_cnt = atoi(optarg);
         break;
@@ -580,6 +589,12 @@ int st_app_parse_args(struct st_app_context* ctx, struct mtl_init_params* p, int
         p->flags &= ~MTL_FLAG_TX_VIDEO_MIGRATE;
         p->flags &= ~MTL_FLAG_RX_VIDEO_MIGRATE;
         break;
+      case ST_ARG_BIND_NUMA:
+        p->flags |= MTL_FLAG_BIND_NUMA;
+        break;
+      case ST_ARG_NOT_BIND_NUMA:
+        p->flags &= ~MTL_FLAG_BIND_NUMA;
+        break;
       case ST_ARG_SHAPING:
         if (!strcmp(optarg, "narrow"))
           ctx->tx_pacing_type = ST21_PACING_NARROW;

diff --git a/app/src/legacy/rx_video_app.c b/app/src/legacy/rx_video_app.c
@@ -563,6 +563,7 @@ static int app_rx_video_init(struct st_app_context* ctx, st_json_video_session_t
     ops.rtcp.nack_interval_us = 250;
   }
   if (ctx->enable_timing_parser) ops.flags |= ST20_RX_FLAG_TIMING_PARSER_STAT;
+  if (ctx->rx_video_multi_thread) ops.flags |= ST20_RX_FLAG_USE_MULTI_THREADS;
   ops.rx_burst_size = ctx->rx_burst_size;
 
   st_pthread_mutex_init(&s->st20_wake_mutex, NULL);

diff --git a/app/src/rx_st20p_app.c b/app/src/rx_st20p_app.c
@@ -224,6 +224,7 @@ static int app_rx_st20p_init(struct st_app_context* ctx,
   ops.flags |= ST20P_RX_FLAG_DMA_OFFLOAD;
   if (st20p && st20p->enable_rtcp) ops.flags |= ST20P_RX_FLAG_ENABLE_RTCP;
   if (ctx->enable_timing_parser) ops.flags |= ST20P_RX_FLAG_TIMING_PARSER_STAT;
+  if (ctx->rx_video_multi_thread) ops.flags |= ST20P_RX_FLAG_USE_MULTI_THREADS;
 
   s->width = ops.width;
   s->height = ops.height;

diff --git a/doc/run.md b/doc/run.md
@@ -415,6 +415,8 @@ packet egresses from the sender.
 --rx_video_file_frames <count>       : debug option, dump the received video frames to one yuv file
 --rx_audio_dump_time_s <seconds>     : debug option, dump the received audio frames to one pcm file
 --dedicated_sys_lcore                : debug option, run MTL system tasks(CNI, PTP, etc...) in a dedicated lcore
+--bind_numa                          : debug option, all MTL threads bind to same numa of NIC
+--not_bind_numa                      : debug option, MTL threads runs without NIC numa aware
 ```
 
 ## 6. Tests

diff --git a/include/mtl_api.h b/include/mtl_api.h
@@ -313,7 +313,7 @@ enum st21_tx_pacing_way {
 
 /** MTL init flag */
 enum mtl_init_flag {
-  /** lib will call numa_bind to bind app thread and memory to NIC socket also.*/
+  /** lib will bind all app and pin threads, memory to NIC socket */
   MTL_FLAG_BIND_NUMA = (MTL_BIT64(0)),
   /** Enable built-in PTP implementation */
   MTL_FLAG_PTP_ENABLE = (MTL_BIT64(1)),

diff --git a/include/st20_api.h b/include/st20_api.h
@@ -200,6 +200,12 @@ extern "C" {
  * Enable the timing analyze info in the st20_rx_frame_meta
  */
 #define ST20_RX_FLAG_TIMING_PARSER_META (MTL_BIT32(22))
+/**
+ * Flag bit in flags of struct st20_rx_ops.
+ * Only for ST20_TYPE_FRAME_LEVEL.
+ * Force to use multi(only two now) threads for the rx packet processing
+ */
+#define ST20_RX_FLAG_USE_MULTI_THREADS (MTL_BIT32(23))
 
 /**
  * Flag bit in flags of struct st22_rx_ops, for non MTL_PMD_DPDK_USER.

diff --git a/include/st_pipeline_api.h b/include/st_pipeline_api.h
@@ -576,6 +576,10 @@ enum st20p_rx_flag {
    * st20p_rx_get_frame
    */
   ST20P_RX_FLAG_TIMING_PARSER_META = (MTL_BIT32(22)),
+  /**
+   * Force to use multi(only two now) threads for the rx packet processing
+   */
+  ST20P_RX_FLAG_USE_MULTI_THREADS = (MTL_BIT32(23)),
 };
 
 /** Bit define for flag_resp of struct st22_decoder_create_req. */

diff --git a/lib/src/mt_main.c b/lib/src/mt_main.c
@@ -604,6 +604,11 @@ mtl_handle mtl_init(struct mtl_init_params* p) {
     }
   }
 
+  if (!(p->flags & MTL_FLAG_BIND_NUMA)) {
+    warn("%s, numa bind is not enabled, performance may limited as across numa access\n",
+         __func__);
+  }
+
   info("%s, succ, tsc_hz %" PRIu64 "\n", __func__, impl->tsc_hz);
   info("%s, simd level %s, flags 0x%" PRIx64 "\n", __func__,
        mtl_get_simd_level_name(mtl_get_simd_level()), p->flags);

diff --git a/lib/src/mt_main.h b/lib/src/mt_main.h
@@ -1431,6 +1431,14 @@ static inline bool mt_user_ptp_service(struct mtl_main_impl* impl) {
     return false;
 }
 
+/* if user enable the numa bind for lcore thread and memory */
+static inline bool mt_user_bind_numa(struct mtl_main_impl* impl) {
+  if (mt_get_user_params(impl)->flags & MTL_FLAG_BIND_NUMA)
+    return true;
+  else
+    return false;
+}
+
 /* if user enable the auto start/stop */
 static inline bool mt_user_auto_start_stop(struct mtl_main_impl* impl) {
   if (mt_get_user_params(impl)->flags & MTL_FLAG_DEV_AUTO_START_STOP)

diff --git a/lib/src/mt_sch.c b/lib/src/mt_sch.c
@@ -640,6 +640,8 @@ int mt_sch_get_lcore(struct mtl_main_impl* impl, unsigned int* lcore,
   bool skip_numa_check = false;
   struct mt_sch_mgr* mgr = mt_sch_get_mgr(impl);
 
+  if (!mt_user_bind_numa(impl)) skip_numa_check = true;
+
 again:
   cur_lcore = 0;
   if (mt_is_manager_connected(impl)) {

diff --git a/lib/src/st2110/pipeline/st20_pipeline_rx.c b/lib/src/st2110/pipeline/st20_pipeline_rx.c
@@ -495,6 +495,8 @@ static int rx_st20p_create_transport(struct mtl_main_impl* impl, struct st20p_rx
     ops_rx.flags |= ST20_RX_FLAG_TIMING_PARSER_STAT;
   if (ops->flags & ST20P_RX_FLAG_TIMING_PARSER_META)
     ops_rx.flags |= ST20_RX_FLAG_TIMING_PARSER_META;
+  if (ops->flags & ST20P_RX_FLAG_USE_MULTI_THREADS)
+    ops_rx.flags |= ST20_RX_FLAG_USE_MULTI_THREADS;
   if (ops->flags & ST20P_RX_FLAG_PKT_CONVERT) {
     uint64_t pkt_cvt_output_cap =
         ST_FMT_CAP_YUV422PLANAR10LE | ST_FMT_CAP_Y210 | ST_FMT_CAP_UYVY;

diff --git a/lib/src/st2110/st_rx_video_session.c b/lib/src/st2110/st_rx_video_session.c
@@ -2426,6 +2426,11 @@ static int rv_init_sw(struct mtl_main_impl* impl, struct st_rx_video_sessions_mg
     if ((bps / (1000 * 1000)) > (40 * 1000)) {
       if (!s->dma_dev) pkt_handle_lcore = true;
     }
+
+    if (ops->flags & ST20_RX_FLAG_USE_MULTI_THREADS) {
+      info("%s(%d), user enable ST20_RX_FLAG_USE_MULTI_THREADS\n", __func__, idx);
+      pkt_handle_lcore = true;
+    }
   }
 
   /* only one core for hdr split mode */
@@ -2437,6 +2442,12 @@ static int rv_init_sw(struct mtl_main_impl* impl, struct st_rx_video_sessions_mg
       rv_uinit_sw(impl, s);
       return -EINVAL;
     }
+    if (ops->num_port > 1) {
+      err("%s(%d), additional pkt lcore not support redundant, num_port %u\n", __func__,
+          idx, ops->num_port);
+      rv_uinit_sw(impl, s);
+      return -EINVAL;
+    }
     ret = rv_init_pkt_lcore(impl, mgr, s);
     if (ret < 0) {
       err("%s(%d), init_pkt_lcore fail %d\n", __func__, idx, ret);

diff --git a/tools/ebpf/et.h b/tools/ebpf/et.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: BSD-3-Clause
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
  * Copyright(c) 2023 Intel Corporation
  */
 

diff --git a/tools/ebpf/lcore_monitor.h b/tools/ebpf/lcore_monitor.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: BSD-3-Clause
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
  * Copyright(c) 2023 Intel Corporation
  */
 

diff --git a/tools/ebpf/udp_monitor.h b/tools/ebpf/udp_monitor.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: BSD-3-Clause
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
  * Copyright(c) 2024 Intel Corporation
  */