Adding Doca Communication Channel API

Mellanox · Nov 28, 2023 · d6b7679 · d6b7679
1 parent 8c23bf7
commit d6b7679
Show file tree

Hide file tree

Showing 10 changed files with 1,142 additions and 76 deletions.
diff --git a/configure.ac b/configure.ac
@@ -10,7 +10,7 @@ AC_CANONICAL_SYSTEM
 
 TARGETDIR="unknown"
 case "$host" in
- 
+
   i?86-*-*) TARGET=X86; TARGETDIR=x86;;
   ia64*-*-*) TARGET=IA64; TARGETDIR=ia64;;
   powerpc*-*-linux* | powerpc-*-sysv*) TARGET=POWERPC; TARGETDIR=powerpc;;
@@ -26,14 +26,14 @@ case "$host" in
   aarch64-*-*) TARGET=AARCH64; TARGETDIR=aarch64;;
   s390*-*-*) TARGET=S390; TARGETDIR=s390;;
 esac
- 
+
 AC_SUBST(AM_RUNTESTFLAGS)
 AC_SUBST(AM_LTLDFLAGS)
- 
+
 if test $TARGETDIR = unknown; then
   AC_MSG_ERROR(["it has not been ported to $host."])
 fi
- 
+
 AM_CONDITIONAL(X86, test x$TARGET = xX86)
 AM_CONDITIONAL(IA64, test x$TARGET = xIA64)
 AM_CONDITIONAL(POWERPC, test x$TARGET = xPOWERPC)
@@ -100,6 +100,34 @@ AC_MSG_CHECKING(
     [for vma extra api])
 AC_MSG_RESULT([${have_vma_api}])
 
+##########################################################################
+# check DOCA communication channel API
+#
+AC_ARG_ENABLE(
+        [doca-communication-channel-api],
+        AC_HELP_STRING([--enable-doca-communication-channel-api],
+                                [SOCKPERF: enable DOCA communication channel extra api support (default=no)]),
+        [have_doca_comm_channel_api=$enableval],
+        [have_doca_comm_channel_api=no])
+AS_IF([test "x${have_doca_comm_channel_api}" == "xyes"],
+        if test "$have_doca_comm_channel_api" = "yes"
+        then
+            have_doca_comm_channel_api=/opt/mellanox/doca
+        fi
+
+        CPPFLAGS="$CPPFLAGS -I$have_doca_comm_channel_api/include -I$have_doca_comm_channel_api/lib -I$have_doca_comm_channel_api/samples"
+        LIBS="$LIBS -ldoca_comm_channel -ldoca_common"
+
+        [AC_CHECK_HEADERS([$have_doca_comm_channel_api/include/doca_comm_channel.h $have_doca_comm_channel_api/include/doca_dev.h $have_doca_comm_channel_api/samples/common.h],
+                [AC_DEFINE([USING_DOCA_COMM_CHANNEL_API],[1],[[Enable using DOCA communication channel extra API]])
+         ],
+                [AC_MSG_ERROR([doca_comm_channel.h file not found])]
+                [have_doca_comm_channel_api=no])])
+AC_MSG_CHECKING(
+        [for doca communication channel extra api])
+AC_MSG_RESULT([${have_doca_comm_channel_api}])
+
+
 ##########################################################################
 # check XLIO extra API
 #
@@ -141,7 +169,7 @@ AM_CONDITIONAL(DOC, test "x$have_doc" = "xyes")
 
 
 ##########################
-# Enable tests 
+# Enable tests
 #
 SP_ARG_ENABLE_BOOL(
     [test],
@@ -151,7 +179,7 @@ AM_CONDITIONAL(TEST, test "x$have_test" = "xyes")
 
 
 ##########################
-# Enable tools 
+# Enable tools
 #
 SP_ARG_ENABLE_BOOL(
     [tool],
@@ -233,6 +261,7 @@ AC_MSG_RESULT([
 	test:		${have_test}
 	tool:		${have_tool}
 	vma_api:	${have_vma_api}
+    doca_api:   ${have_doca_comm_channel_api}
 	xlio_api:	${have_xlio_api}
 	debug:		${have_debug}
 ])
diff --git a/src/client.cpp b/src/client.cpp
@@ -885,7 +885,11 @@ int Client<IoType, SwitchCycleDuration, PongModeCare>::initBeforeLoop() {
             if (!(data && (data->active_fd_list))) continue;
 
             const sockaddr_store_t *p_client_bind_addr = &g_pApp->m_const_params.client_bind_info;
-            if (p_client_bind_addr->addr.sa_family != AF_UNSPEC) {
+#if defined(USING_DOCA_COMM_CHANNEL_API)
+            if (p_client_bind_addr->addr.sa_family != AF_UNSPEC && !s_user_params.doca_comm_channel) {
+#else
+             if (p_client_bind_addr->addr.sa_family != AF_UNSPEC) {
+#endif //USING_DOCA_COMM_CHANNEL_API
                 socklen_t client_bind_addr_len = g_pApp->m_const_params.client_bind_info_len;
                 std::string hostport = sockaddr_to_hostport(p_client_bind_addr);
 #if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)

diff --git a/src/common.cpp b/src/common.cpp
@@ -64,7 +64,11 @@ std::string sockaddr_to_hostport(const struct sockaddr *addr)
     if (addr->sa_family == AF_INET6) {
         return "[" + std::string(hbuf) + "]:" + std::string(pbuf);
     } else if (addr->sa_family == AF_UNIX) {
-        return std::string(addr->sa_data);
+#if defined(USING_DOCA_COMM_CHANNEL_API)
+        if (s_user_params.doca_comm_channel)
+            return std::string(pbuf) + " [DOCA]";
+#endif
+        return std::string(pbuf) + " [UNIX]";
     } else {
         return std::string(hbuf) + ":" + std::string(pbuf);
     }

diff --git a/src/common.h b/src/common.h
@@ -28,6 +28,7 @@
 
 #ifndef COMMON_H_
 #define COMMON_H_
+#define POLL_TIMEOUT_MS -1
 
 #include <string>
 
@@ -112,6 +113,65 @@ static inline int msg_sendto(int fd, uint8_t *buf, int nbytes,
             ret = tls_write(g_fds_array[fd]->tls_handle, buf, nbytes);
         } else
 #endif /* DEFINED_TLS */
+#if defined(USING_DOCA_COMM_CHANNEL_API)
+        if (g_fds_array[fd]->doca_objects) {
+            doca_error_t doca_error;
+            struct doca_cc_send_task *task;
+            struct doca_task *task_obj;
+            union doca_data user_data;
+            struct cc_ctrl_path_objects *sample_objects = g_fds_array[fd]->doca_objects;
+            struct timespec ts = {
+                .tv_sec = 0,
+                .tv_nsec = SLEEP_IN_NANOS,
+            };
+
+            if (s_user_params.mode == MODE_SERVER) {
+                /* This function will only be called after a message was received, so connection should be available */
+                if (sample_objects->connection == NULL) {
+		            DOCA_LOG_ERR("Failed to send response: no connection available");
+		            return DOCA_ERROR_NOT_CONNECTED;
+	            }
+                doca_error = doca_cc_server_send_task_alloc_init(sample_objects->server, sample_objects->connection, buf,
+                                                                nbytes, &task);
+                if (doca_error != DOCA_SUCCESS) {
+                    DOCA_LOG_ERR("Failed to allocate task in server with error = %s", doca_error_get_name(doca_error));
+                    return doca_error;
+                }
+            } else { // MODE_CLIENT
+                // need to call doca_pe_progress here once, to move client to state DOCA_CTX_STATE_RUNNING
+                if (sample_objects->state != DOCA_CTX_STATE_RUNNING) {
+                    log_dbg("msg_sendto: calling to doca_pe_progress");
+                    while (doca_pe_progress(sample_objects->pe) == 0) {
+                        nanosleep(&ts, &ts);
+                    }
+                }
+                doca_error = doca_cc_client_send_task_alloc_init(sample_objects->client, sample_objects->connection, buf, nbytes, &task);
+                if (doca_error != DOCA_SUCCESS) {
+                    DOCA_LOG_ERR("Failed to allocate task in client with error = %s", doca_error_get_name(doca_error));
+                    return doca_error;
+                }
+            }
+
+            task_obj = doca_cc_send_task_as_task(task);
+            if (s_user_params.mode == MODE_SERVER) {
+                user_data.ptr = (void *)sample_objects;
+                doca_task_set_user_data(task_obj, user_data);
+            }
+
+            doca_error = doca_task_submit(task_obj);
+            if (doca_error != DOCA_SUCCESS) {
+                DOCA_LOG_ERR("Failed submitting send task with error = %s", doca_error_get_name(doca_error));
+                doca_task_free(task_obj);
+                return doca_error;
+            }
+
+            while (doca_pe_progress(sample_objects->pe) == 0) {
+                    nanosleep(&ts, &ts);
+            }
+
+            ret = nbytes;
+        } else
+#endif /* USING_DOCA_COMM_CHANNEL_API */
         {
             ret = sendto(fd, buf, nbytes, flags, sendto_addr, addrlen);
         }

diff --git a/src/defs.h b/src/defs.h
@@ -28,7 +28,6 @@
 
 #ifndef DEFS_H_
 #define DEFS_H_
-
 #define __STDC_FORMAT_MACROS
 
 #ifdef __windows__
@@ -157,6 +156,11 @@ typedef unsigned short int sa_family_t;
 #endif // USING_XLIO_EXTRA_API
 #endif // !defined(__windows__) && !defined(__FreeBSD__) && !defined(__APPLE__)
 
+#ifdef USING_DOCA_COMM_CHANNEL_API
+#include "doca_cc_helper.h"
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#endif /* USING_DOCA_COMM_CHANNEL_API */
+
 #define MIN_PAYLOAD_SIZE (MsgHeader::EFFECTIVE_SIZE)
 extern int MAX_PAYLOAD_SIZE;
 extern int max_fds_num;
@@ -291,8 +295,14 @@ enum {
     OPT_LOAD_XLIO,                // 47
     OPT_TCP_NB_CONN_TIMEOUT_MS,   // 48
 #if defined(DEFINED_TLS)
-    OPT_TLS
+    OPT_TLS,
 #endif /* DEFINED_TLS */
+#if defined(USING_DOCA_COMM_CHANNEL_API)
+    OPT_DOCA,
+    OPT_PCI,
+    OPT_PCI_REP
+#endif /* USING_DOCA_COMM_CHANNEL_API */
+
 };
 
 static const char *const round_trip_str[] = { "latency", "rtt" };
@@ -571,6 +581,9 @@ struct fds_data {
 #if defined(DEFINED_TLS)
     void *tls_handle = nullptr;
 #endif /* DEFINED_TLS */
+#if defined(USING_DOCA_COMM_CHANNEL_API)
+    struct cc_ctrl_path_objects *doca_objects = nullptr;
+#endif /* USING_DOCA_COMM_CHANNEL_API */
 
     fds_data()
     {
@@ -804,6 +817,11 @@ struct user_params_t {
 #if defined(DEFINED_TLS)
     bool tls = false;
 #endif /* DEFINED_TLS */
+#if defined(USING_DOCA_COMM_CHANNEL_API)
+    bool doca_comm_channel = false;
+    char cc_dev_pci_addr[PCI_ADDR_LEN];                       /* Comm Channel DOCA device PCI address */
+    char cc_dev_rep_pci_addr[PCI_ADDR_LEN];                 /* Comm Channel DOCA device representor PCI address */
+#endif /* USING_DOCA_COMM_CHANNEL_API */
 
     user_params_t() {
         memset(&client_bind_info, 0, sizeof(client_bind_info));