Added support for XRC, fixed bugs

Author: Johann George

AUTHORS

@@ -4,3 +4,4 @@ Thanks to the following for their contributions:
     Dotan Barak
     Ralph Campbell
     Yevgeny Kliteynik
+    Dave Olson

configure.in

@@ -1,5 +1,5 @@
-AC_INIT(qperf, 0.4.1, [email protected])
-AM_INIT_AUTOMAKE(qperf, 0.4.1)
+AC_INIT(qperf, 0.4.2, [email protected])
+AM_INIT_AUTOMAKE(qperf, 0.4.2)
 AC_PROG_CC
 AC_CHECK_LIB(ibverbs, ibv_open_device, RDMA=1)
 AC_CHECK_LIB(rdmacm, rdma_create_id)

qperf.spec

@@ -1,6 +1,6 @@
 Name:           qperf
 Summary:        Measure socket and RDMA performance
-Version:        0.4.1
+Version:        0.4.2
 Release:        1
 License:        BSD 3-Clause, GPL v2
 Group:          Networking/Diagnostic

src/help.txt

@@ -94,6 +94,9 @@ Categories +RDMA
         udp_lat
         ver_rc_compare_swap
         ver_rc_fetch_add
+        xrc_bi_bw
+        xrc_bw
+        xrc_lat
 Examples
     In these examples, we first run qperf on a node called myserver in server
     mode by invoking it with no arguments.  In all the subsequent examples, we
@@ -111,12 +114,15 @@ Examples
             qperf myserver rc_bi_bw
 Opts
     --access_recv OnOff (-ar)           Turn on/off accessing received data
-      -aro                              Cause received data to be accessed
+      -ar1                              Cause received data to be accessed
+    --alt_port Port (-ap)               Set alternate path port
+      --loc_alt_port Port (-lap)        Set local alternate path port
+      --rem_alt_port Port (-rap)        Set remote alternate path port
     --cpu_affinity PN (-ca)             Set processor affinity
       --loc_cpu_affinity PN (-lca)      Set local processor affinity
       --rem_cpu_affinity PN (-rca)      Set remote processor affinity
     --flip OnOff (-f)                   Flip on/off sender and receiver
-      -fo                               Flip (on) sender and receiver
+      -f1                               Flip (on) sender and receiver
     --help Topic (-h)                   Get more information on a topic
     --host Node (-H)                    Identify server node
     --id Device:Port (-i)               Set RDMA device and port
@@ -129,9 +135,9 @@ Opts
     --cq_poll OnOff                     Set polling mode on/off
       --loc_cq_poll OnOff (-lcp)        Set local polling mode on/off
       --rem_cq_poll OnOff (-rcp)        Set remote polling mode on/off
-      -cpo                              Turn polling mode on
-      -lcpo                             Turn local polling mode on
-      -rcpo                             Turn remote polling mode on
+      -cp1                              Turn polling mode on
+      -lcp1                             Turn local polling mode on
+      -rcp1                             Turn remote polling mode on
     --ip_port Port (-ip)                Set TCP port used for tests
     --precision Digits (-e)             Set precision reported
     --rd_atomic Max (-nr)               Set RDMA read/atomic count
@@ -154,7 +160,7 @@ Opts
     --unify_units (-uu)                 Unify units
     --use_bits_per_sec (-ub)            Use bits/sec rather than bytes/sec
     --use_cm OnOff (-cm)                Use RDMA Connection Manager or not
-      -cmo                              Use RDMA Connection Manager
+      -cm1                              Use RDMA Connection Manager
     --verbose (-v)                      Verbose; turn on all of -v[cstu]
       --verbose_conf (-vc)              Show configuration information
       --verbose_stat (-vs)              Show statistical information
@@ -172,8 +178,14 @@ Options
           If OnOff is non-zero, data is accessed once received.  Otherwise,
           data is ignored.  By default, OnOff is 0.  This can help to mimic
           some applications.
-      -aro
+      -ar1
           Cause received data to be accessed.
+    --alt_port Port (-ap)
+          Set alternate path port. This enables automatic path failover.
+      --loc_alt_port Port (-lap)
+          Set local alternate path port. This enables automatic path failover.
+      --rem_alt_port Port (-rap)
+          Set remote alternate path port. This enables automatic path failover.
     --cpu_affinity PN (-ca)
           Set cpu affinity to PN.  CPUs are numbered sequentially from 0.  If
           PN is "any", any cpu is allowed otherwise the cpu is limited to the
@@ -184,7 +196,7 @@ Options
           Set remote processor affinity to PN.
     --flip OnOff (-f)
           If non-zero, cause sender and receiver to play opposite roles.
-      -fo
+      -f1
           Cause sender and receiver to play opposite roles.
     --help Topic (-h)
           Print out information about Topic.  To see the list of topics, type
@@ -222,11 +234,11 @@ Options
           Locally turn polling mode on or off.
       --rem_cq_poll OnOff (-rcp)
           Remotely turn polling mode on or off.
-      -cpo
+      -cp1
           Turn polling mode on.
-      -lcpo
+      -lcp1
           Turn local polling mode on.
-      -rcpo
+      -rcp1
           Turn remote polling mode on.
     --ip_port Port (-ip)
           Use Port to run the socket tests.  This is different from
@@ -294,7 +306,7 @@ Options
           necessary to use the CM for iWARP devices.  The default is to
           establish the connection without using the CM.  This only works for
           the tests that use the RC transport.
-      -cmo
+      -cm1
           Use RDMA Connection Manager.
     --verbose (-v)
           Provide more detailed output.  Turns on -vc, -vs, -vt and -vu.
@@ -894,3 +906,52 @@ ver_rc_fetch_add +RDMA
         Tests the RC Fetch and Add Atomic operation.  The server's memory
         location starts with zero and the client successively adds one.  The
         results are checked for correctness.
+xrc_bw +RDMA
+    Purpose
+        XRC streaming one way bandwidth
+    Common Options
+        --access_recv OnOff (-ar)   Access received data
+        --id Device:Port (-i)       Set RDMA device and port
+        --msg_size Size (-m)        Set message size
+        --cq_poll OnOff             Set polling mode on/off
+        --time (-t)                 Set test duration
+    Other Options
+        --cpu_affinity, --listen_port, --mtu_size, --static_rate, --timeout
+    Display Options
+        --precision, --unify_nodes, --unify_units, --use_bits_per_sec,
+        --verbose
+    Description
+        The client sends messages to the server who notes how many it received.
+        The XRC Send/Receive mechanism is used.
+xrc_bi_bw +RDMA
+    Purpose
+        XRC streaming two way bandwidth
+    Common Options
+        --access_recv OnOff (-ar)   Access received data
+        --id Device:Port (-i)       Set RDMA device and port
+        --msg_size Size (-m)        Set message size
+        --cq_poll OnOff             Set polling mode on/off
+        --time (-t)                 Set test duration
+    Other Options
+        --cpu_affinity, --listen_port, --mtu_size, --static_rate, --timeout
+    Display Options
+        --precision, --unify_nodes, --unify_units, --use_bits_per_sec,
+        --verbose
+    Description
+        Both the client and server exchange messages with each other using the
+        XRC Send/Receive mechanism and note how many were received.
+xrc_lat +RDMA
+    Purpose
+        XRC one way latency
+    Common Options
+        --id Device:Port (-i)   Set RDMA device and port
+        --msg_size Size (-m)    Set message size
+        --cq_poll OnOff         Set polling mode on/off
+        --time (-t)             Set test duration
+    Other Options
+        --cpu_affinity, --listen_port, --mtu_size, --static_rate, --timeout
+    Display Options
+        --precision, --unify_nodes, --unify_units, --verbose
+    Description
+        A ping pong latency test where the server and client exchange messages
+        repeatedly using XRC Send/Receive.

src/qperf.c

@@ -62,7 +62,7 @@
  */
 #define VER_MAJ 0                       /* Major version */
 #define VER_MIN 4                       /* Minor version */
-#define VER_INC 1                       /* Incremental version */
+#define VER_INC 2                       /* Incremental version */
 #define LISTENQ 5                       /* Size of listen queue */
 #define BUFSIZE 1024                    /* Size of buffers */
 
@@ -273,6 +273,7 @@ volatile int Finished;
 PAR_NAME ParName[] ={
     { "access_recv",    L_ACCESS_RECV,    R_ACCESS_RECV   },
     { "affinity",       L_AFFINITY,       R_AFFINITY      },
+    { "alt_port",       L_ALT_PORT,       R_ALT_PORT      },
     { "flip",           L_FLIP,           R_FLIP          },
     { "id",             L_ID,             R_ID            },
     { "msg_size",       L_MSG_SIZE,       R_MSG_SIZE      },
@@ -299,6 +300,8 @@ PAR_INFO ParInfo[P_N] ={
     { R_ACCESS_RECV,    'l',  &RReq.access_recv     },
     { L_AFFINITY,       'l',  &Req.affinity         },
     { R_AFFINITY,       'l',  &RReq.affinity        },
+    { L_ALT_PORT,       'l',  &Req.alt_port         },
+    { R_ALT_PORT,       'l',  &RReq.alt_port        },
     { L_FLIP,           'l',  &Req.flip             },
     { R_FLIP,           'l',  &RReq.flip            },
     { L_ID,             'p',  &Req.id               },
@@ -378,6 +381,13 @@ DICT Renamed[] = {
     {   "-vS",              "-vvs",                 },
     {   "-vT",              "-vvt",                 },
     {   "-vU",              "-vvu",                 },
+    /* options that are on */
+    {   "-aro",             "-ar1"                  },
+    {   "-cmo",             "-cm1"                  },
+    {   "-fo",              "-f1"                   },
+    {   "-cpo",             "-cp1"                  },
+    {   "-lcpo",            "-lcp1"                 },
+    {   "-rcpo",            "-rcp1"                 },
     /* miscellaneous */
     {   "-Ar",              "-ar"                   },
     {   "-M",               "-mt"                   },
@@ -393,7 +403,13 @@ DICT Renamed[] = {
 OPTION Options[] ={
     { "--access_recv",        "int",   L_ACCESS_RECV,   R_ACCESS_RECV   },
     {   "-ar",                "int",   L_ACCESS_RECV,   R_ACCESS_RECV   },
-    {   "-aro",               "set1",  L_ACCESS_RECV,   R_ACCESS_RECV   },
+    {   "-ar1",               "set1",  L_ACCESS_RECV,   R_ACCESS_RECV   },
+    { "--alt_port",           "int",   L_ALT_PORT,      R_ALT_PORT      },
+    {   "-ap",                "int",   L_ALT_PORT,      R_ALT_PORT      },
+    {  "--loc_alt_port",      "int",   L_ALT_PORT,                      },
+    {   "-lap",               "int",   L_ALT_PORT,                      },
+    {  "--rem_alt_port",      "int",   R_ALT_PORT                       },
+    {   "-rap",               "int",   R_ALT_PORT                       },
     { "--cpu_affinity",       "int",   L_AFFINITY,      R_AFFINITY      },
     {   "-ca",                "int",   L_AFFINITY,      R_AFFINITY      },
     {  "--loc_cpu_affinity",  "int",   L_AFFINITY,                      },
@@ -404,7 +420,7 @@ OPTION Options[] ={
     {   "-D",                 "Sdebug",                                 },
     { "--flip",               "int",   L_FLIP,          R_FLIP          },
     {   "-f",                 "int",   L_FLIP,          R_FLIP          },
-    {   "-fo",                "set1",  L_FLIP,          R_FLIP          },
+    {   "-f1",                "set1",  L_FLIP,          R_FLIP          },
     { "--help",               "help"                                    }, 
     {   "-h",                 "help"                                    }, 
     { "--host",               "host",                                   },
@@ -425,13 +441,13 @@ OPTION Options[] ={
     {   "-n",                 "int",   L_NO_MSGS,       R_NO_MSGS       },
     { "--cq_poll",            "int",   L_POLL_MODE,     R_POLL_MODE     },
     {  "-cp",                 "int",   L_POLL_MODE,     R_POLL_MODE     },
-    {   "-cpo",               "set1",  L_POLL_MODE,     R_POLL_MODE     },
+    {   "-cp1",               "set1",  L_POLL_MODE,     R_POLL_MODE     },
     {  "--loc_cq_poll",       "int",   L_POLL_MODE,                     },
     {   "-lcp",               "int",   L_POLL_MODE,                     },
-    {   "-lcpo",              "set1",  L_POLL_MODE                      },
+    {   "-lcp1",              "set1",  L_POLL_MODE                      },
     {  "--rem_cq_poll",       "int",   R_POLL_MODE                      },
     {   "-rcp",               "int",   R_POLL_MODE                      },
-    {   "-rcpo",              "set1",  R_POLL_MODE                      },
+    {   "-rcp1",              "set1",  R_POLL_MODE                      },
     { "--ip_port",            "int",   L_PORT,          R_PORT          },
     {   "-ip",                "int",   L_PORT,          R_PORT          },
     { "--precision",          "precision",                              },
@@ -476,7 +492,7 @@ OPTION Options[] ={
     {   "-ub",                "ub",                                     },
     { "--use_cm",             "int",   L_USE_CM,        R_USE_CM        },
     {   "-cm",                "int",   L_USE_CM,        R_USE_CM        },
-    {   "-cmo",               "set1",  L_USE_CM,        R_USE_CM        },
+    {   "-cm1",               "set1",  L_USE_CM,        R_USE_CM        },
     { "--verbose",            "v",                                      },
     {   "-v",                 "v",                                      },
     { "--verbose_conf",       "vc",                                     },
@@ -543,6 +559,9 @@ TEST Tests[] ={
     test(ud_lat),
     test(ver_rc_compare_swap),
     test(ver_rc_fetch_add),
+    test(xrc_bi_bw),
+    test(xrc_bw),
+    test(xrc_lat),
 #endif
 };
 
@@ -786,6 +805,7 @@ do_option(OPTION *option, char ***argvp)
         /* Help */
         char **usage;
         char *category = (*argvp)[1];
+
         if (!category)
             category = "main";
         for (usage = Usage; *usage; usage += 2)
@@ -1137,7 +1157,7 @@ server(void)
         TEST *test;
         int s = offset(REQ, req_index);
 
-        debug("waiting for request");
+        debug("ready for requests");
         if (!server_recv_request())
             continue;
         pid = fork();
@@ -1164,7 +1184,7 @@ server(void)
 
         test = &Tests[Req.req_index];
         TestName = test->name;
-        debug("request is %s", TestName);
+        debug("received request: %s", TestName);
         init_lstat();
         Finished = 0;
         set_affinity();
@@ -1283,7 +1303,7 @@ client(TEST *test)
     RReq.ver_inc = VER_INC;
     RReq.req_index = test - Tests;
     TestName = test->name;
-    debug("sending request %s", TestName);
+    debug("sending request: %s", TestName);
     init_lstat();
     printf("%s:\n", TestName);
     Finished = 0;
@@ -1378,12 +1398,12 @@ exchange_results(void)
         recv_mesg(&stat, sizeof(stat), "results");
         dec_init(&stat);
         dec_stat(&RStat);
-        send_sync("results");
+        send_sync("synchronization after test");
     } else {
         enc_init(&stat);
         enc_stat(&LStat);
         send_mesg(&stat, sizeof(stat), "results");
-        recv_sync("results");
+        recv_sync("synchronization after test");
     }
 }
 
@@ -1583,7 +1603,7 @@ run_server_quit(void)
 void
 sync_test(void)
 {
-    synchronize("test");
+    synchronize("synchronization before test");
     start_test_timer(Req.time);
 }
 
@@ -1601,7 +1621,7 @@ start_test_timer(int seconds)
     if (!seconds)
         return;
 
-    debug("starting timer");
+    debug("starting timer for %d seconds", seconds);
     itimerval.it_value.tv_sec = seconds;
     itimerval.it_interval.tv_usec = 1;
     setitimer(ITIMER_REAL, &itimerval, 0);
@@ -2422,6 +2442,7 @@ enc_req(REQ *host)
     enc_int(host->req_index,     sizeof(host->req_index));
     enc_int(host->access_recv,   sizeof(host->access_recv));
     enc_int(host->affinity,      sizeof(host->affinity));
+    enc_int(host->alt_port,      sizeof(host->alt_port));
     enc_int(host->flip,          sizeof(host->flip));
     enc_int(host->msg_size,      sizeof(host->msg_size));
     enc_int(host->mtu_size,      sizeof(host->mtu_size));
@@ -2462,6 +2483,7 @@ dec_req_data(REQ *host)
     host->req_index     = dec_int(sizeof(host->req_index));
     host->access_recv   = dec_int(sizeof(host->access_recv));
     host->affinity      = dec_int(sizeof(host->affinity));
+    host->alt_port      = dec_int(sizeof(host->alt_port));
     host->flip          = dec_int(sizeof(host->flip));
     host->msg_size      = dec_int(sizeof(host->msg_size));
     host->mtu_size      = dec_int(sizeof(host->mtu_size));

src/qperf.h

@@ -49,7 +49,7 @@
 #define cardof(a)       (sizeof(a)/sizeof(*a))
 #define endof(a)        (&a[cardof(a)])
 #define streq(a, b)     (strcmp(a, b) == 0)
-#define offset(t, e)    ((int)&((t *)0)->e)
+#define offset(t, e)    ((long)&((t *)0)->e)
 #define is_client()     (ServerName != 0)
 #define is_sender()     (Req.flip ? !is_client() : is_client())
 
@@ -97,6 +97,8 @@ typedef enum {
     R_ACCESS_RECV,
     L_AFFINITY,
     R_AFFINITY,
+    L_ALT_PORT,
+    R_ALT_PORT,
     L_FLIP,
     R_FLIP,
     L_ID,
@@ -153,6 +155,7 @@ typedef struct REQ {
     uint16_t    req_index;              /* Request index (into Tests) */
     uint32_t    access_recv;            /* Access data after receiving */
     uint32_t    affinity;               /* Processor affinity */
+    uint32_t    alt_port;               /* Alternate path port number */
     uint32_t    flip;                   /* Flip sender/receiver */
     uint32_t    msg_size;               /* Message Size */
     uint32_t    mtu_size;               /* MTU Size */
@@ -343,6 +346,12 @@ void    run_client_ver_rc_compare_swap(void);
 void    run_server_ver_rc_compare_swap(void);
 void    run_client_ver_rc_fetch_add(void);
 void    run_server_ver_rc_fetch_add(void);
+void    run_client_xrc_bi_bw(void);
+void    run_server_xrc_bi_bw(void);
+void    run_client_xrc_bw(void);
+void    run_server_xrc_bw(void);
+void    run_client_xrc_lat(void);
+void    run_server_xrc_lat(void);
 
 
 /*