st22: add interlace support

test with st22p_interlaced_pcap.json and st22p_1v_1080i59.json

Signed-off-by: Frank Du <[email protected]>

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Changelog for 24.04
 
 * dpdk: upgrade dpdk version to 23.11.
+* st22: add interlaced support.
 
 ## Changelog for 23.12
 

app/src/parse_json.c

@@ -1166,6 +1166,13 @@ static int parse_st22p_fps(json_object* st22p_obj, st_json_st22p_session_t* st22
   return ST_JSON_SUCCESS;
 }
 
+static int parse_st22p_interlace(json_object* st22p_obj, st_json_st22p_session_t* st22p) {
+  json_object* obj = st_json_object_object_get(st22p_obj, "interlaced");
+  if (!obj) return ST_JSON_SUCCESS;
+  st22p->info.interlaced = json_object_get_boolean(obj);
+  return ST_JSON_SUCCESS;
+}
+
 static int parse_st22p_pack_type(json_object* st22p_obj, st_json_st22p_session_t* st22p) {
   const char* pack_type =
       json_object_get_string(st_json_object_object_get(st22p_obj, "pack_type"));
@@ -1312,6 +1319,10 @@ static int st_json_parse_tx_st22p(int idx, json_object* st22p_obj,
   ret = parse_st22p_fps(st22p_obj, st22p);
   if (ret < 0) return ret;
 
+  /* parse interlace */
+  ret = parse_st22p_interlace(st22p_obj, st22p);
+  if (ret < 0) return ret;
+
   /* parse pack_type */
   ret = parse_st22p_pack_type(st22p_obj, st22p);
   if (ret < 0) return ret;
@@ -1382,6 +1393,10 @@ static int st_json_parse_rx_st22p(int idx, json_object* st22p_obj,
   ret = parse_st22p_fps(st22p_obj, st22p);
   if (ret < 0) return ret;
 
+  /* parse interlace */
+  ret = parse_st22p_interlace(st22p_obj, st22p);
+  if (ret < 0) return ret;
+
   /* parse pack_type */
   ret = parse_st22p_pack_type(st22p_obj, st22p);
   if (ret < 0) return ret;

app/src/parse_json.h

@@ -181,6 +181,7 @@ typedef struct st_json_st22p_info {
   uint32_t width;
   uint32_t height;
   enum st_fps fps;
+  bool interlaced;
   enum st_plugin_device device;
   enum st22_codec codec;
   enum st22_pack_type pack_type;

app/src/rx_st20p_app.c

@@ -31,6 +31,10 @@ static void app_rx_st20p_consume_frame(struct st_app_rx_st20p_session* s,
            frame->pkts_recv[MTL_SESSION_PORT_R], frame->pkts_total);
   }
 
+  if (frame->interlaced) {
+    dbg("%s(%d), %s field\n", __func__, s->idx, frame->second_field ? "second" : "first");
+  }
+
   if (d && d->front_frame) {
     if (st_pthread_mutex_trylock(&d->display_frame_mutex) == 0) {
       if (frame->fmt == ST_FRAME_FMT_YUV422RFC4175PG2BE10) {

app/src/rx_st22p_app.c

@@ -18,6 +18,10 @@ static void app_rx_st22p_consume_frame(struct st_app_rx_st22p_session* s,
                                        struct st_frame* frame) {
   struct st_display* d = s->display;
 
+  if (frame->interlaced) {
+    dbg("%s(%d), %s field\n", __func__, s->idx, frame->second_field ? "second" : "first");
+  }
+
   if (d && d->front_frame) {
     if (st_pthread_mutex_trylock(&d->display_frame_mutex) == 0) {
       if (frame->fmt == ST_FRAME_FMT_UYVY)
@@ -163,6 +167,7 @@ static int app_rx_st22p_init(struct st_app_context* ctx,
   ops.width = st22p ? st22p->info.width : 1920;
   ops.height = st22p ? st22p->info.height : 1080;
   ops.fps = st22p ? st22p->info.fps : ST_FPS_P59_94;
+  ops.interlaced = st22p ? st22p->info.interlaced : false;
   ops.output_fmt = st22p ? st22p->info.format : ST_FRAME_FMT_YUV422RFC4175PG2BE10;
   ops.port.payload_type = st22p ? st22p->base.payload_type : ST_APP_PAYLOAD_TYPE_ST22;
   ops.pack_type = st22p ? st22p->info.pack_type : ST22_PACK_CODESTREAM;

app/src/tx_st20p_app.c

@@ -62,6 +62,10 @@ static void app_tx_st20p_build_frame(struct st_app_tx_st20p_session* s,
   /* point to next frame */
   s->st20p_frame_cursor += s->st20p_frame_size;
 
+  if (frame->interlaced) {
+    dbg("%s(%d), %s field\n", __func__, s->idx, frame->second_field ? "second" : "first");
+  }
+
   app_tx_st20p_display_frame(s, frame);
 }
 

app/src/tx_st22p_app.c

@@ -48,6 +48,10 @@ static void app_tx_st22p_build_frame(struct st_app_tx_st22p_session* s,
   /* point to next frame */
   s->st22p_frame_cursor += s->st22p_frame_size;
 
+  if (frame->interlaced) {
+    dbg("%s(%d), %s field\n", __func__, s->idx, frame->second_field ? "second" : "first");
+  }
+
   app_tx_st22p_display_frame(s, frame);
 }
 
@@ -239,13 +243,15 @@ static int app_tx_st22p_init(struct st_app_context* ctx, st_json_st22p_session_t
   ops.width = st22p ? st22p->info.width : 1920;
   ops.height = st22p ? st22p->info.height : 1080;
   ops.fps = st22p ? st22p->info.fps : ST_FPS_P59_94;
+  ops.interlaced = st22p ? st22p->info.interlaced : false;
   ops.input_fmt = st22p ? st22p->info.format : ST_FRAME_FMT_YUV422RFC4175PG2BE10;
   ops.pack_type = st22p ? st22p->info.pack_type : ST22_PACK_CODESTREAM;
   ops.codec = st22p ? st22p->info.codec : ST22_CODEC_JPEGXS;
   ops.device = st22p ? st22p->info.device : ST_PLUGIN_DEVICE_AUTO;
   ops.quality = st22p ? st22p->info.quality : ST22_QUALITY_MODE_SPEED;
   ops.codec_thread_cnt = st22p ? st22p->info.codec_thread_count : 0;
   ops.codestream_size = ops.width * ops.height * 3 / 8;
+  if (ops.interlaced) ops.codestream_size /= 2; /* the size is for each field */
   ops.framebuff_cnt = 2;
   ops.notify_frame_available = app_tx_st22p_frame_available;
   if (st22p && st22p->enable_rtcp) ops.flags |= ST22P_TX_FLAG_ENABLE_RTCP;

app/src/tx_video_app.c

@@ -298,6 +298,7 @@ static void* app_tx_video_pcap_thread(void* arg) {
     }
     udp_data_len = 0;
     packet = (uint8_t*)pcap_next(s->st20_pcap, &hdr);
+    dbg("%s(%d), packet %p\n", __func__, idx, packet);
     if (packet) {
       eth_hdr = (struct ether_header*)packet;
       if (ntohs(eth_hdr->ether_type) == ETHERTYPE_IP) {
@@ -306,6 +307,7 @@ static void* app_tx_video_pcap_thread(void* arg) {
           udp_hdr =
               (struct udphdr*)(packet + sizeof(struct ether_header) + sizeof(struct ip));
           udp_data_len = ntohs(udp_hdr->len) - sizeof(struct udphdr);
+          dbg("%s(%d), packet %p udp_data_len %u\n", __func__, idx, packet, udp_data_len);
           mtl_memcpy(usrptr,
                      packet + sizeof(struct ether_header) + sizeof(struct ip) +
                          sizeof(struct udphdr),
@@ -324,7 +326,12 @@ static void* app_tx_video_pcap_thread(void* arg) {
     }
 
     struct st_rfc3550_rtp_hdr* hdr = (struct st_rfc3550_rtp_hdr*)usrptr;
-    if (hdr->payload_type != s->payload_type) udp_data_len = 0;
+    if (hdr->payload_type != s->payload_type) {
+      udp_data_len = 0;
+      err("%s(%d), expect payload_type %u but pcap is %u, please correct the "
+          "payload_type in json\n",
+          __func__, idx, s->payload_type, hdr->payload_type);
+    }
 
     st20_tx_put_mbuf(s->handle, mbuf, udp_data_len);
 

include/st20_api.h

@@ -499,6 +499,8 @@ struct st22_tx_frame_meta {
   uint32_t height;
   /** Frame resolution fps, set by lib */
   enum st_fps fps;
+  /** Second field type indicate, for interlaced mode */
+  bool second_field;
   /** codestream_size for next_frame_idx, set by user */
   size_t codestream_size;
   /** Timestamp format, user can customize it if ST22_TX_FLAG_USER_PACING */
@@ -513,6 +515,8 @@ struct st22_tx_frame_meta {
  * Frame meta data of st2110-22(video) rx streaming
  */
 struct st22_rx_frame_meta {
+  /** Second field type indicate, for interlaced mode */
+  bool second_field;
   /** Frame timestamp format */
   enum st10_timestamp_fmt tfmt;
   /** Frame timestamp value */
@@ -572,7 +576,8 @@ MTL_PACK(struct st22_rfc9134_rtp_hdr {
   struct st_rfc3550_rtp_hdr base;
   /** F counter high part */
   uint8_t f_counter_hi : 3;
-  /** Interlaced information */
+  /** Interlaced information, 0b00: progressively, 0b10: first field, 0b11: second field.
+   */
   uint8_t interlaced : 2;
   /** Last */
   uint8_t last_packet : 1;
@@ -975,6 +980,8 @@ struct st20_tx_ops {
   uint32_t height;
   /** Mandatory. Session resolution fps */
   enum st_fps fps;
+  /** Mandatory. interlaced or not */
+  bool interlaced;
   /** Mandatory. Session resolution format */
   enum st20_fmt fmt;
   /** Mandatory. 7 bits payload type defined in RFC3550 */
@@ -990,13 +997,6 @@ struct st20_tx_ops {
   /** Optional. Flags to control session behaviors. See ST20_TX_FLAG_* for possible value
    */
   uint32_t flags;
-  /** Optional.
-   * interlace or not, false(default): non-interlaced: true: interlaced.
-   * Ex for format 1080i50, please refer to below parameter configurations:
-   *   interlaced: true, width: 1920, height: 1080, fps: ST_FPS_P50
-   * and filled each frame(field) with 540 lines.
-   */
-  bool interlaced;
 
   /**
    * Mandatory for ST20_TYPE_FRAME_LEVEL/ST20_TYPE_SLICE_LEVEL.
@@ -1121,6 +1121,8 @@ struct st22_tx_ops {
   uint32_t height;
   /** Mandatory. Session resolution fps */
   enum st_fps fps;
+  /** Mandatory. interlaced or not */
+  bool interlaced;
   /** Mandatory. Session resolution format */
   enum st20_fmt fmt;
   /** Mandatory. 7 bits payload type define in RFC3550 */
@@ -1292,6 +1294,8 @@ struct st20_rx_ops {
   uint32_t height;
   /** Mandatory. Session resolution fps */
   enum st_fps fps;
+  /** Mandatory. interlaced or not */
+  bool interlaced;
   /** Mandatory. Session resolution format */
   enum st20_fmt fmt;
   /** Mandatory. 7 bits payload type define in RFC3550 */
@@ -1312,13 +1316,6 @@ struct st20_rx_ops {
   /** Optional. Flags to control session behaviors. See ST20_RX_FLAG_* for possible value
    */
   uint32_t flags;
-  /** Optional.
-   * interlace or not, false(default): non-interlaced: true: interlaced.
-   * Ex for format 1080i50, please refer to below parameter configurations:
-   *   interlaced: true, width: 1920, height: 1080, fps: ST_FPS_P50
-   * and each frame(field) received has 540 lines data.
-   */
-  bool interlaced;
 
   /**
    * Mandatory for ST20_TYPE_FRAME_LEVEL/ST20_TYPE_SLICE_LEVEL.
@@ -1443,6 +1440,8 @@ struct st22_rx_ops {
   uint32_t height;
   /** Mandatory. Session resolution fps */
   enum st_fps fps;
+  /** Mandatory. interlaced or not */
+  bool interlaced;
   /** Mandatory. Session resolution format */
   enum st20_fmt fmt;
   /** Mandatory. 7 bits payload type define in RFC3550 */

include/st_pipeline_api.h

@@ -551,6 +551,8 @@ struct st22_encoder_create_req {
   uint32_t height;
   /** Session resolution fps, set by lib */
   enum st_fps fps;
+  /** Interlaced or not, set by lib */
+  bool interlaced;
   /** Session input frame format, set by lib */
   enum st_frame_fmt input_fmt;
   /** Session output frame format, set by lib */
@@ -606,6 +608,8 @@ struct st22_decoder_create_req {
   uint32_t height;
   /** Session resolution fps, set by lib */
   enum st_fps fps;
+  /** Interlaced or not, set by lib */
+  bool interlaced;
   /** Session input frame format, set by lib */
   enum st_frame_fmt input_fmt;
   /** Session output frame format, set by lib */
@@ -640,9 +644,9 @@ struct st22_decoder_dev {
 
 /** The structure info for st22 decode frame meta. */
 struct st22_decode_frame_meta {
-  /** Encode source frame */
+  /** Decode source frame */
   struct st_frame* src;
-  /** Encode dst frame */
+  /** Decode dst frame */
   struct st_frame* dst;
   /** priv pointer for lib, do not touch this */
   void* priv;
@@ -743,6 +747,8 @@ struct st20p_tx_ops {
   uint32_t height;
   /** Mandatory. Session resolution fps */
   enum st_fps fps;
+  /** Mandatory. interlaced or not */
+  bool interlaced;
   /** Mandatory. Session input frame format */
   enum st_frame_fmt input_fmt;
   /** Mandatory. Session transport pacing type */
@@ -778,13 +784,7 @@ struct st20p_tx_ops {
    * tasklet routine.
    */
   int (*notify_frame_done)(void* priv, struct st_frame* frame);
-  /** Optional.
-   * interlace or not, false(default): non-interlaced: true: interlaced.
-   * Ex for format 1080i50, please refer to below parameter configurations:
-   *   interlaced: true, width: 1920, height: 1080, fps: ST_FPS_P50
-   * and filled each frame(field) with 540 lines.
-   */
-  bool interlaced;
+
   /** Optional. Linesize for transport frame, only for non-convert mode */
   size_t transport_linesize;
 
@@ -836,6 +836,8 @@ struct st20p_rx_ops {
   uint32_t height;
   /** Mandatory. Session resolution fps */
   enum st_fps fps;
+  /** Mandatory. interlaced or not */
+  bool interlaced;
   /** Mandatory. Session transport frame format */
   enum st20_fmt transport_fmt;
   /** Mandatory. Session output frame format */
@@ -862,13 +864,6 @@ struct st20p_rx_ops {
    */
   int (*notify_frame_available)(void* priv);
 
-  /** Optional.
-   * interlace or not, false(default): non-interlaced: true: interlaced.
-   * Ex for format 1080i50, please refer to below parameter configurations:
-   *   interlaced: true, width: 1920, height: 1080, fps: ST_FPS_P50
-   * and each frame(field) received has 540 lines data.
-   */
-  bool interlaced;
   /** Optional. Linesize for transport frame, only for non-convert mode */
   size_t transport_linesize;
   /** Optional. Array of external frames */
@@ -901,6 +896,8 @@ struct st22p_tx_ops {
   uint32_t height;
   /** Mandatory. Session resolution fps */
   enum st_fps fps;
+  /** Mandatory. interlaced or not */
+  bool interlaced;
   /** Mandatory. Session input frame format */
   enum st_frame_fmt input_fmt;
   /** Mandatory. packetization modes define in RFC9134 */
@@ -911,7 +908,8 @@ struct st22p_tx_ops {
   enum st_plugin_device device;
   /** Mandatory. speed or quality mode */
   enum st22_quality_mode quality;
-  /** Mandatory. codestream size, calculate as compress ratio */
+  /** Mandatory. codestream size, calculate as compress ratio. For interlaced, it's the
+   * expected codestream size for each field */
   size_t codestream_size;
   /**
    *  Mandatory. the frame buffer count requested for one st22 pipeline tx session,
@@ -967,6 +965,8 @@ struct st22p_rx_ops {
   uint32_t height;
   /** Mandatory. Session resolution fps */
   enum st_fps fps;
+  /** Mandatory. interlaced or not */
+  bool interlaced;
   /** Mandatory. Session output frame format */
   enum st_frame_fmt output_fmt;
   /** Mandatory. packetization modes define in RFC9134 */
@@ -990,7 +990,8 @@ struct st22p_rx_ops {
   uint32_t flags;
   /** Optional. thread count for codec, leave to zero if not know */
   uint32_t codec_thread_cnt;
-  /** Optional. max codestream size, lib will use output frame size if not set */
+  /** Optional. max codestream size, lib will use output frame size if not set. For
+   * interlaced, it's the expected codestream size for each field */
   size_t max_codestream_size;
   /** Optional for ST22P_RX_FLAG_ENABLE_RTCP. RTCP info */
   struct st_rx_rtcp_ops* rtcp;

lib/src/st2110/pipeline/st20_pipeline_tx.c

@@ -481,7 +481,7 @@ struct st_frame* st20p_tx_get_frame(st20p_tx_handle handle) {
 
   dbg("%s(%d), frame %u succ\n", __func__, idx, framebuff->idx);
   struct st_frame* frame = tx_st20p_user_frame(ctx, framebuff);
-  if (ctx->ops.interlaced) { /* init second_field but user still can customize also */
+  if (ctx->ops.interlaced) { /* init second_field but user still can customize */
     frame->second_field = ctx->second_field;
     ctx->second_field = ctx->second_field ? false : true;
   }
@@ -527,6 +527,10 @@ int st20p_tx_put_frame(st20p_tx_handle handle, struct st_frame* frame) {
     framebuff->user_meta_data_size = frame->user_meta_size;
   }
 
+  if (ctx->ops.interlaced) { /* update second_field */
+    framebuff->dst.second_field = framebuff->src.second_field = frame->second_field;
+  }
+
   if (ctx->internal_converter) { /* convert internal */
     ctx->internal_converter->convert_func(&framebuff->src, &framebuff->dst);
     framebuff->stat = ST20P_TX_FRAME_CONVERTED;
@@ -565,6 +569,10 @@ int st20p_tx_put_ext_frame(st20p_tx_handle handle, struct st_frame* frame,
     return -EIO;
   }
 
+  if (ctx->ops.interlaced) { /* update second_field */
+    framebuff->dst.second_field = framebuff->src.second_field = frame->second_field;
+  }
+
   uint8_t planes = st_frame_fmt_planes(framebuff->src.fmt);
   if (ctx->derive) {
     struct st20_ext_frame trans_ext_frame;