diff --git a/.clang-format b/.clang-format index e26bf0be..3f0bc3bc 100644 --- a/.clang-format +++ b/.clang-format @@ -1,26 +1,25 @@ -BasedOnStyle : google -SpaceBeforeParens : Always -IndentWidth : 4 -BreakBeforeBraces : Linux +BasedOnStyle: Google +AlignAfterOpenBracket: Align +AlignOperands: 'true' +AlignTrailingComments: 'true' +AllowAllParametersOfDeclarationOnNextLine: 'false' +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: 'false' +BinPackArguments: 'false' +BinPackParameters: 'false' +BreakBeforeBinaryOperators: NonAssignment +BreakBeforeBraces: Linux +BreakBeforeTernaryOperators: 'true' +ColumnLimit: '90' +ConstructorInitializerAllOnOneLineOrOnePerLine: 'true' +IndentWidth: '4' +PenaltyBreakBeforeFirstCallParameter: '10000000' +PenaltyBreakString: '10' +PenaltyReturnTypeOnItsOwnLine: '10000000' +PointerAlignment: Right +SpaceBeforeParens: Always +TabWidth: '4' UseTab: Never -AllowShortIfStatementsOnASingleLine : false -ConstructorInitializerAllOnOneLineOrOnePerLine : true -AllowShortFunctionsOnASingleLine : false -AllowShortLoopsOnASingleLine : false -BinPackParameters : false -AllowAllParametersOfDeclarationOnNextLine : false -AlignTrailingComments : true -ColumnLimit : 80 - -# do not put all arguments on one line unless it's the same line as the call -PenaltyBreakBeforeFirstCallParameter : 10000000 -PenaltyReturnTypeOnItsOwnLine : 65000 -PenaltyBreakString : 10 - -# These improve formatting results but require clang 3.6/7 or higher -BreakBeforeBinaryOperators : NonAssignment -AlignAfterOpenBracket: true -BinPackArguments : true -AlignOperands : true -BreakBeforeTernaryOperators : true -AllowAllParametersOfDeclarationOnNextLine : false +AlwaysBreakAfterReturnType: None +AlwaysBreakAfterDefinitionReturnType: None \ No newline at end of file diff --git a/configure.ac b/configure.ac index a81029d5..371f3c95 100644 --- a/configure.ac +++ b/configure.ac @@ -75,15 +75,47 @@ AC_ARG_ENABLE([perfflow], # TODO Add support for libb64 back once base64 encoding/decoding is fully complete # AC_ARG_VAR([LIBB64_DIR], [root directory for libb64]) +############################################# +# Define PKG_CHECK_VAR if it does not exist # +############################################# + +# Macro is copied from https://github.com/pkgconf/pkgconf/blob/master/pkg.m4 +# with minor modifications to change comments from using 'dnl' to '#' +m4_ifndef([PKG_CHECK_VAR], [ +# PKG_CHECK_VAR(VARIABLE, MODULE, CONFIG-VARIABLE, +# [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +# ------------------------------------------- +# Since: 0.28 +# +# Retrieves the value of the pkg-config variable for the given module. +AC_DEFUN([PKG_CHECK_VAR], +[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl +AC_ARG_VAR([$1], [value of $3 for $2, overriding pkg-config])dnl +_PKG_CONFIG([$1], [variable="][$3]["], [$2]) +AS_VAR_COPY([$1], [pkg_cv_][$1]) +AS_VAR_IF([$1], [""], [$5], [$4])dnl +])# End of PKG_CHECK_VAR +]) + ######################## # Checks for libraries # ######################## # Check for the dl library (specifically, the "dlsym" function) AC_CHECK_LIB([dl], [dlsym]) -# FIXME: Replace 'main' with a function in '-ldyad_fstream': -# AC_CHECK_LIB([dyad_fstream], [main]) -# FIXME: Replace 'main' with a function in '-ltap': -# AC_CHECK_LIB([tap], [main]) +# Check for UCX v1.6.0 or higher (Required) +# TODO: make UCX optional based on a new AC_ARG_ENABLE flag +PKG_CHECK_MODULES([UCX], + [ucx >= 1.6.0] +) +PKG_CHECK_VAR([UCX_LIBDIR], + [ucx >= 1.6.0], + [libdir], + [], + [AC_MSG_FAILURE([Could not find libdir for UCX])] +) +AS_IF([test "x$UCX_LIBDIR" = "x"], + [AC_MSG_FAILURE([check_var succeeded, but value is incorrect])] +) # Find and get info for Flux-Core using pkg-config AX_FLUX_CORE PKG_CHECK_MODULES([JANSSON], @@ -153,7 +185,8 @@ AC_CHECK_FUNCS( \ # Add any necessary compilation flags # ####################################### if test "x$enable_debug" = xyes; then - CPPFLAGS="$CPPFLAGS -DDYAD_FULL_DEBUG=1 -DDYAD_LOGGING_ON=1" + CFLAGS="$CFLAGS -DDYAD_FULL_DEBUG=1 -DDYAD_LOGGING_ON=1" + CXXFLAGS="$CXXFLAGS -DDYAD_FULL_DEBUG=1 -DDYAD_LOGGING_ON=1" fi ######################## @@ -162,7 +195,9 @@ fi AC_CONFIG_FILES([Makefile src/Makefile src/utils/Makefile + src/utils/base64/Makefile src/utils/libtap/Makefile + src/dtl/Makefile src/core/Makefile src/stream/Makefile src/modules/Makefile diff --git a/src/Makefile.am b/src/Makefile.am index 06bca034..626467d4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1 +1 @@ -SUBDIRS = utils modules core wrapper stream +SUBDIRS = utils dtl modules core wrapper stream diff --git a/src/core/Makefile.am b/src/core/Makefile.am index 1c0a8916..e34a0b29 100644 --- a/src/core/Makefile.am +++ b/src/core/Makefile.am @@ -1,11 +1,29 @@ lib_LTLIBRARIES = libdyad_core.la -libdyad_core_la_SOURCES = dyad_core.c -libdyad_core_la_LIBADD = $(top_builddir)/src/utils/libutils.la $(top_builddir)/src/utils/libmurmur3.la $(FLUX_CORE_LIBS) -libdyad_core_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_builddir)/src/utils $(FLUX_CORE_CFLAGS) -libdyad_core_la_LDFLAGS = -export-symbols dyad_core.sym $(AM_LDFLAGS) +libdyad_core_la_SOURCES = \ + dyad_core.c +libdyad_core_la_LIBADD = \ + $(top_builddir)/src/dtl/libdyad_dtl.la \ + $(UCX_LIBS) \ + $(JANSSON_LIBS) \ + $(FLUX_CORE_LIBS) +libdyad_core_la_CFLAGS = \ + $(AM_CFLAGS) \ + -I$(top_srcdir)/src/utils \ + -I$(top_srcdir)/src/utils/base64 \ + -I$(top_srcdir)/src/dtl \ + $(UCX_CFLAGS) \ + $(JANSSON_CFLAGS) \ + $(FLUX_CORE_CFLAGS) \ + -DBUILDING_DYAD=1 \ + -fvisibility=hidden +libdyad_core_la_CPPFLAGS = +libdyad_core_la_LDFLAGS = \ + -Wl,-rpath,'$(UCX_LIBDIR)' \ + $(AM_LDFLAGS) if PERFFLOW libdyad_core_la_LIBADD += $(PERFFLOW_LIBS) -libdyad_core_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS) $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1 +libdyad_core_la_CFLAGS += $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1 +libdyad_core_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS) endif -include_HEADERS = dyad_core.h dyad_envs.h dyad_rc.h dyad_flux_log.h +include_HEADERS = dyad_core.h dyad_envs.h diff --git a/src/core/dyad_core.c b/src/core/dyad_core.c index 0db18ea7..018fc995 100644 --- a/src/core/dyad_core.c +++ b/src/core/dyad_core.c @@ -1,8 +1,9 @@ +#include "dyad_core.h" + #include #include -#include "dyad_core.h" -#include "dyad_flux_log.h" +#include "dyad_dtl_impl.h" #include "murmur3.h" #include "utils.h" @@ -14,14 +15,20 @@ #include #endif +#if DYAD_PERFFLOW +#define DYAD_CORE_FUNC_MODS __attribute__ ((annotate ("@critical_path()"))) static +#else +#define DYAD_CORE_FUNC_MODS static inline +#endif + const struct dyad_ctx dyad_ctx_default = { NULL, // h + NULL, // dtl_handle false, // debug false, // check false, // reenter true, // initialized false, // shared_storage - false, // sync_started 3u, // key_depth 1024u, // key_bins 0u, // rank @@ -30,15 +37,28 @@ const struct dyad_ctx dyad_ctx_default = { NULL // cons_managed_path }; +struct dyad_kvs_response { + char* fpath; + uint32_t owner_rank; +}; +typedef struct dyad_kvs_response dyad_kvs_response_t; + static int gen_path_key (const char* str, char* path_key, const size_t len, const uint32_t depth, const uint32_t width) { - static const uint32_t seeds[10] = {104677u, 104681u, 104683u, 104693u, - 104701u, 104707u, 104711u, 104717u, - 104723u, 104729u}; + static const uint32_t seeds[10] = {104677u, + 104681u, + 104683u, + 104693u, + 104701u, + 104707u, + 104711u, + 104717u, + 104723u, + 104729u}; uint32_t seed = 57u; uint32_t hash[4] = {0u}; // Output for the hash @@ -68,14 +88,7 @@ static int gen_path_key (const char* str, return 0; } -#if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) -static dyad_rc_t dyad_kvs_commit (const dyad_ctx_t* ctx, - flux_kvs_txn_t* txn) -#else -static inline dyad_rc_t dyad_kvs_commit (const dyad_ctx_t* ctx, - flux_kvs_txn_t* txn) -#endif +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_kvs_commit (const dyad_ctx_t* ctx, flux_kvs_txn_t* txn) { flux_future_t* f = NULL; DYAD_LOG_INFO (ctx, "Committing transaction to KVS\n"); @@ -94,22 +107,14 @@ static inline dyad_rc_t dyad_kvs_commit (const dyad_ctx_t* ctx, return DYAD_RC_OK; } -#if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) -static dyad_rc_t publish_via_flux (const dyad_ctx_t* restrict ctx, - const char* restrict upath) -#else -static inline dyad_rc_t publish_via_flux (const dyad_ctx_t* restrict ctx, - const char* restrict upath) -#endif +DYAD_CORE_FUNC_MODS dyad_rc_t publish_via_flux (const dyad_ctx_t* restrict ctx, + const char* restrict upath) { dyad_rc_t rc = DYAD_RC_OK; - const char* prod_managed_path = NULL; flux_kvs_txn_t* txn = NULL; const size_t topic_len = PATH_MAX; char topic[PATH_MAX + 1]; memset (topic, 0, topic_len + 1); - prod_managed_path = ctx->prod_managed_path; memset (topic, '\0', topic_len + 1); // Generate the KVS key from the file path relative to // the producer-managed directory @@ -146,14 +151,8 @@ publish_done:; return rc; } -#if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) -static dyad_rc_t dyad_commit (const dyad_ctx_t* restrict ctx, - const char* restrict fname) -#else -static inline dyad_rc_t dyad_commit (dyad_ctx_t* restrict ctx, - const char* restrict fname) -#endif +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_commit (dyad_ctx_t* restrict ctx, + const char* restrict fname) { dyad_rc_t rc = DYAD_RC_OK; char upath[PATH_MAX]; @@ -161,16 +160,12 @@ static inline dyad_rc_t dyad_commit (dyad_ctx_t* restrict ctx, // Extract the path to the file specified by fname relative to the // producer-managed path // This relative path will be stored in upath - if (!cmp_canonical_path_prefix (ctx->prod_managed_path, fname, upath, - PATH_MAX)) { - DYAD_LOG_INFO (ctx, "%s is not in the Producer's managed path\n", - fname); + if (!cmp_canonical_path_prefix (ctx->prod_managed_path, fname, upath, PATH_MAX)) { + DYAD_LOG_INFO (ctx, "%s is not in the Producer's managed path\n", fname); rc = DYAD_RC_OK; goto commit_done; } - DYAD_LOG_INFO (ctx, - "Obtained file path relative to producer directory: %s\n", - upath); + DYAD_LOG_INFO (ctx, "Obtained file path relative to producer directory: %s\n", upath); // Call publish_via_flux to actually store information about the file into // the Flux KVS // Fence this call with reassignments of reenter so that, if intercepting @@ -188,27 +183,17 @@ commit_done:; return rc; } -#if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) -static dyad_rc_t dyad_kvs_lookup (const dyad_ctx_t* ctx, - const char* restrict kvs_topic, - uint32_t* owner_rank, - flux_future_t** f) -#else -static inline dyad_rc_t dyad_kvs_lookup (const dyad_ctx_t* ctx, - const char* restrict kvs_topic, - uint32_t* owner_rank, - flux_future_t** f) -#endif +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_kvs_lookup (const dyad_ctx_t* ctx, + const char* restrict kvs_topic, + uint32_t* owner_rank, + flux_future_t** f) { dyad_rc_t rc = DYAD_RC_OK; // Lookup information about the desired file (represented by kvs_topic) // from the Flux KVS. If there is no information, wait for it to be // made available - DYAD_LOG_INFO (ctx, "Retrieving information from KVS under the key %s\n", - kvs_topic); - *f = flux_kvs_lookup (ctx->h, ctx->kvs_namespace, FLUX_KVS_WAITCREATE, - kvs_topic); + DYAD_LOG_INFO (ctx, "Retrieving information from KVS under the key %s\n", kvs_topic); + *f = flux_kvs_lookup (ctx->h, ctx->kvs_namespace, FLUX_KVS_WAITCREATE, kvs_topic); // If the KVS lookup failed, log an error and return DYAD_BADLOOKUP if (*f == NULL) { DYAD_LOG_ERR (ctx, "KVS lookup failed!\n"); @@ -225,16 +210,9 @@ static inline dyad_rc_t dyad_kvs_lookup (const dyad_ctx_t* ctx, return DYAD_RC_OK; } -#if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) -static dyad_rc_t dyad_fetch (const dyad_ctx_t* restrict ctx, - const char* restrict fname, - dyad_kvs_response_t** restrict resp) -#else -static inline dyad_rc_t dyad_fetch (const dyad_ctx_t* restrict ctx, - const char* restrict fname, - dyad_kvs_response_t** restrict resp) -#endif +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_fetch (const dyad_ctx_t* restrict ctx, + const char* restrict fname, + dyad_kvs_response_t** restrict resp) { dyad_rc_t rc = DYAD_RC_OK; char upath[PATH_MAX]; @@ -247,15 +225,11 @@ static inline dyad_rc_t dyad_fetch (const dyad_ctx_t* restrict ctx, // Extract the path to the file specified by fname relative to the // consumer-managed path // This relative path will be stored in upath - if (!cmp_canonical_path_prefix (ctx->cons_managed_path, fname, upath, - PATH_MAX)) { - DYAD_LOG_INFO (ctx, "%s is not in the Consumer's managed path\n", - fname); + if (!cmp_canonical_path_prefix (ctx->cons_managed_path, fname, upath, PATH_MAX)) { + DYAD_LOG_INFO (ctx, "%s is not in the Consumer's managed path\n", fname); return DYAD_RC_OK; } - DYAD_LOG_INFO (ctx, - "Obtained file path relative to consumer directory: %s\n", - upath); + DYAD_LOG_INFO (ctx, "Obtained file path relative to consumer directory: %s\n", upath); // Generate the KVS key from the file path relative to // the consumer-managed directory gen_path_key (upath, topic, topic_len, ctx->key_depth, ctx->key_bins); @@ -287,8 +261,7 @@ static inline dyad_rc_t dyad_fetch (const dyad_ctx_t* restrict ctx, // Allocate and populate the dyad_kvs_response_t object. // If an error occurs, log it and return the appropriate // return code - DYAD_LOG_INFO (ctx, - "Creating KVS response object to store retrieved data\n"); + DYAD_LOG_INFO (ctx, "Creating KVS response object to store retrieved data\n"); *resp = malloc (sizeof (struct dyad_kvs_response)); if (*resp == NULL) { DYAD_LOG_ERR (ctx, "Cannot allocate a dyad_kvs_response_t object!\n"); @@ -296,6 +269,14 @@ static inline dyad_rc_t dyad_fetch (const dyad_ctx_t* restrict ctx, goto fetch_done; } (*resp)->fpath = malloc (strlen (upath) + 1); + if ((*resp)->fpath == NULL) { + DYAD_LOG_ERR (ctx, + "Cannot allocate a buffer for the file path in the " + "dyad_kvs_response_t object\n"); + free (*resp); + rc = DYAD_RC_BADRESPONSE; + goto fetch_done; + } strncpy ((*resp)->fpath, upath, strlen (upath) + 1); (*resp)->owner_rank = owner_rank; rc = DYAD_RC_OK; @@ -308,75 +289,112 @@ fetch_done:; return rc; } -#if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) -static dyad_rc_t dyad_rpc_get ( - const dyad_ctx_t* ctx, - const dyad_kvs_response_t* restrict kvs_data, - const char** file_data, - int* file_len, - flux_future_t** f) -#else -static inline dyad_rc_t dyad_rpc_get ( - const dyad_ctx_t* ctx, - const dyad_kvs_response_t* restrict kvs_data, - const char** file_data, - int* file_len, - flux_future_t** f) -#endif +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_get_data (const dyad_ctx_t* ctx, + const dyad_kvs_response_t* restrict kvs_data, + const char** file_data, + size_t* file_len) { dyad_rc_t rc = DYAD_RC_OK; - // Create and send an RPC payload to the producer's Flux broker. - // This payload will tell the broker to run the dyad.fetch function - // with the upath specified by the data fetched from the KVS - DYAD_LOG_INFO (ctx, "Sending RPC for data for %s from rank %u\n", - kvs_data->fpath, kvs_data->owner_rank); - *f = flux_rpc_pack (ctx->h, "dyad.fetch", kvs_data->owner_rank, 0, "{s:s}", - "upath", kvs_data->fpath); - // If the RPC dispatch failed, log an error and return DYAD_BADRPC - if (*f == NULL) { - DYAD_LOG_ERR (ctx, "Cannot send RPC to producer plugin!\n"); - return DYAD_RC_BADRPC; + dyad_rc_t final_rc = DYAD_RC_OK; + flux_future_t* f; + json_t* rpc_payload; + DYAD_LOG_INFO (ctx, "Packing payload for RPC to DYAD module"); + rc = ctx->dtl_handle->rpc_pack (ctx->dtl_handle, + kvs_data->fpath, + kvs_data->owner_rank, + &rpc_payload); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERR (ctx, + "Cannot create JSON payload for Flux RPC to DYAD " + "module\n"); + goto get_done; + } + DYAD_LOG_INFO (ctx, "Sending payload for RPC to DYAD module"); + f = flux_rpc_pack (ctx->h, + DYAD_DTL_RPC_NAME, + kvs_data->owner_rank, + FLUX_RPC_STREAMING, + "o", + rpc_payload); + if (f == NULL) { + DYAD_LOG_ERR (ctx, "Cannot send RPC to producer module\n"); + rc = DYAD_RC_BADRPC; + goto get_done; } - DYAD_LOG_INFO (ctx, "Retrieving data from producer\n"); - // Extract the file data from the RPC response - rc = flux_rpc_get_raw (*f, (const void**)file_data, file_len); - // If the extraction failed, log an error and return DYAD_BADRPC - if (rc < 0) { - DYAD_LOG_ERR (ctx, "Cannot get file back from plugin via RPC!\n"); - return DYAD_RC_BADRPC; + DYAD_LOG_INFO (ctx, "Receive RPC response from DYAD module"); + rc = ctx->dtl_handle->rpc_recv_response (ctx->dtl_handle, f); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERR (ctx, "Cannot receive and/or parse the RPC response\n"); + goto get_done; } - return DYAD_RC_OK; + DYAD_LOG_INFO (ctx, "Establish DTL connection with DYAD module"); + rc = ctx->dtl_handle->establish_connection (ctx->dtl_handle, DYAD_COMM_RECV); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERR (ctx, + "Cannot establish connection with DYAD module on broker " + "%u\n", + kvs_data->owner_rank); + goto get_done; + } + DYAD_LOG_INFO (ctx, "Receive file data via DTL"); + rc = ctx->dtl_handle->recv (ctx->dtl_handle, (void**)file_data, file_len); + DYAD_LOG_INFO (ctx, "Close DTL connection with DYAD module"); + ctx->dtl_handle->close_connection (ctx->dtl_handle); + if (DYAD_IS_ERROR (rc)) { + DYAD_LOG_ERR (ctx, "Cannot receive data from producer module\n"); + goto get_done; + } + + rc = DYAD_RC_OK; + +get_done:; + // There are two return codes that have special meaning when coming from the + // DTL: + // * DYAD_RC_RPC_FINISHED: occurs when an ENODATA error occurs + // * DYAD_RC_BADRPC: occurs when a previous RPC operation fails + // In either of these cases, we do not need to wait for the end of stream + // because the RPC is already completely messed up. If we do not have either + // of these cases, we will wait for one more RPC message. If everything went + // well in the module, this last message will set errno to ENODATA (i.e., + // end of stream). Otherwise, something went wrong, so we'll return + // DYAD_RC_BADRPC. + DYAD_LOG_INFO (ctx, + "Wait for end-of-stream message from module (current RC = %d)\n", + rc); + if (rc != DYAD_RC_RPC_FINISHED && rc != DYAD_RC_BADRPC) { + if (!(flux_rpc_get (f, NULL) < 0 && errno == ENODATA)) { + DYAD_LOG_ERR (ctx, + "An error occured at end of getting data! Either the " + "module sent too many responses, or the module " + "failed with a bad error (errno = %d)\n", + errno); + rc = DYAD_RC_BADRPC; + } + } + DYAD_LOG_INFO (ctx, "Destroy the Flux future for the RPC\n"); + flux_future_destroy (f); + return rc; } -#if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) -static dyad_rc_t dyad_pull (const dyad_ctx_t* restrict ctx, - const char* restrict fname, - const dyad_kvs_response_t* restrict kvs_data) -#else -static inline dyad_rc_t dyad_pull (const dyad_ctx_t* restrict ctx, - const char* restrict fname, - const dyad_kvs_response_t* restrict kvs_data) -#endif +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_pull (const dyad_ctx_t* restrict ctx, + const dyad_kvs_response_t* restrict kvs_data) { dyad_rc_t rc = DYAD_RC_OK; const char* file_data = NULL; - int file_len = 0; + size_t file_len = 0; const char* odir = NULL; FILE* of = NULL; char file_path[PATH_MAX + 1]; char file_path_copy[PATH_MAX + 1]; mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); size_t written_len = 0; - flux_future_t* f = NULL; memset (file_path, 0, PATH_MAX + 1); memset (file_path_copy, 0, PATH_MAX + 1); - // Call dyad_rpc_get to dispatch a RPC to the producer's Flux broker + // Call dyad_get_data to dispatch a RPC to the producer's Flux broker // and retrieve the data associated with the file - rc = dyad_rpc_get (ctx, kvs_data, &file_data, &file_len, &f); - if (rc != DYAD_RC_OK) { + rc = dyad_get_data (ctx, kvs_data, &file_data, &file_len); + if (DYAD_IS_ERROR (rc)) { goto pull_done; } @@ -389,8 +407,7 @@ static inline dyad_rc_t dyad_pull (const dyad_ctx_t* restrict ctx, // Create the directory as needed // TODO: Need to be consistent with the mode at the source odir = dirname (file_path_copy); - if ((strncmp (odir, ".", strlen (".")) != 0) - && (mkdir_as_needed (odir, m) < 0)) { + if ((strncmp (odir, ".", strlen (".")) != 0) && (mkdir_as_needed (odir, m) < 0)) { DYAD_LOG_ERR (ctx, "Cannot create needed directories for pulled " "file\n"); @@ -420,10 +437,8 @@ static inline dyad_rc_t dyad_pull (const dyad_ctx_t* restrict ctx, rc = DYAD_RC_OK; pull_done: - // Destroy the Flux future for the RPC, if needed - if (f != NULL) { - flux_future_destroy (f); - f = NULL; + if (file_data != NULL) { + free (file_data); } // If "check" is set and the operation was successful, set the // DYAD_CHECK_ENV environment variable to "ok" @@ -440,8 +455,10 @@ dyad_rc_t dyad_init (bool debug, const char* kvs_namespace, const char* prod_managed_path, const char* cons_managed_path, + dyad_dtl_mode_t dtl_mode, dyad_ctx_t** ctx) { + dyad_rc_t rc = DYAD_RC_OK; // If ctx is NULL, we won't be able to return a dyad_ctx_t // to the user. In that case, print an error and return // immediately with DYAD_NOCTX. @@ -498,11 +515,13 @@ dyad_rc_t dyad_init (bool debug, } // Get the rank of the Flux broker corresponding // to the handle. If this fails, return DYAD_FLUXFAIL + FLUX_LOG_INFO ((*ctx)->h, "DYAD_CORE: getting Flux rank"); if (flux_get_rank ((*ctx)->h, &((*ctx)->rank)) < 0) { FLUX_LOG_ERR ((*ctx)->h, "Could not get Flux rank!\n"); return DYAD_RC_FLUXFAIL; } // If the namespace is provided, copy it into the dyad_ctx_t object + FLUX_LOG_INFO ((*ctx)->h, "DYAD_CORE: saving KVS namespace"); if (kvs_namespace == NULL) { FLUX_LOG_ERR ((*ctx)->h, "No KVS namespace provided!\n"); // TODO see if we want a different return val @@ -511,15 +530,23 @@ dyad_rc_t dyad_init (bool debug, const size_t namespace_len = strlen (kvs_namespace); (*ctx)->kvs_namespace = (char*)malloc (namespace_len + 1); if ((*ctx)->kvs_namespace == NULL) { - FLUX_LOG_ERR ((*ctx)->h, - "Could not allocate buffer for KVS namespace!\n"); + FLUX_LOG_ERR ((*ctx)->h, "Could not allocate buffer for KVS namespace!\n"); free (*ctx); *ctx = NULL; return DYAD_RC_NOCTX; } strncpy ((*ctx)->kvs_namespace, kvs_namespace, namespace_len + 1); + // Initialize the DTL based on the value of dtl_mode + // If an error occurs, log it and return an error + FLUX_LOG_INFO ((*ctx)->h, "DYAD_CORE: inintializing DYAD DTL"); + rc = dyad_dtl_init (&(*ctx)->dtl_handle, dtl_mode, (*ctx)->h, (*ctx)->debug); + if (DYAD_IS_ERROR (rc)) { + FLUX_LOG_ERR ((*ctx)->h, "Cannot initialize the DTL\n"); + return rc; + } // If the producer-managed path is provided, copy it into // the dyad_ctx_t object + FLUX_LOG_INFO ((*ctx)->h, "DYAD_CORE: saving producer path"); if (prod_managed_path == NULL) { (*ctx)->prod_managed_path = NULL; } else { @@ -534,11 +561,11 @@ dyad_rc_t dyad_init (bool debug, *ctx = NULL; return DYAD_RC_NOCTX; } - strncpy ((*ctx)->prod_managed_path, prod_managed_path, - prod_path_len + 1); + strncpy ((*ctx)->prod_managed_path, prod_managed_path, prod_path_len + 1); } // If the consumer-managed path is provided, copy it into // the dyad_ctx_t object + FLUX_LOG_INFO ((*ctx)->h, "DYAD_CORE: saving consumer path"); if (cons_managed_path == NULL) { (*ctx)->cons_managed_path = NULL; } else { @@ -554,8 +581,7 @@ dyad_rc_t dyad_init (bool debug, *ctx = NULL; return DYAD_RC_NOCTX; } - strncpy ((*ctx)->cons_managed_path, cons_managed_path, - cons_path_len + 1); + strncpy ((*ctx)->cons_managed_path, cons_managed_path, cons_path_len + 1); } // Initialization is now complete! // Set reenter and initialized to indicate this. @@ -565,6 +591,109 @@ dyad_rc_t dyad_init (bool debug, return DYAD_RC_OK; } +dyad_rc_t dyad_init_env (dyad_ctx_t** ctx) +{ + char* e = NULL; + bool debug = false; + bool check = false; + bool shared_storage = false; + unsigned int key_depth = 0; + unsigned int key_bins = 0; + char* kvs_namespace = NULL; + char* prod_managed_path = NULL; + char* cons_managed_path = NULL; + size_t dtl_mode_env_len = 0; + dyad_dtl_mode_t dtl_mode = DYAD_DTL_UCX; + + if ((e = getenv (DYAD_SYNC_DEBUG_ENV))) { + debug = true; + enable_debug_dyad_utils (); + } else { + debug = false; + disable_debug_dyad_utils (); + } + + if (debug) + fprintf (stderr, + "DYAD_CORE: Initializing with environment " + "variables\n"); + + if ((e = getenv (DYAD_SYNC_CHECK_ENV))) { + check = true; + } else { + check = false; + } + + if ((e = getenv (DYAD_SHARED_STORAGE_ENV))) { + shared_storage = true; + } else { + shared_storage = false; + } + + if ((e = getenv (DYAD_KEY_DEPTH_ENV))) { + key_depth = atoi (e); + } else { + key_depth = 3; + } + + if ((e = getenv (DYAD_KEY_BINS_ENV))) { + key_bins = atoi (e); + } else { + key_bins = 1024; + } + + if ((e = getenv (DYAD_KVS_NAMESPACE_ENV))) { + kvs_namespace = e; + } else { + kvs_namespace = NULL; + } + + if ((e = getenv (DYAD_PATH_CONSUMER_ENV))) { + cons_managed_path = e; + } else { + cons_managed_path = NULL; + } + + if ((e = getenv (DYAD_PATH_PRODUCER_ENV))) { + prod_managed_path = e; + } else { + prod_managed_path = NULL; + } + + if ((e = getenv (DYAD_DTL_MODE_ENV))) { + dtl_mode_env_len = strlen (e); + if (strncmp (e, "FLUX_RPC", dtl_mode_env_len) == 0) { + dtl_mode = DYAD_DTL_FLUX_RPC; + } else if (strncmp (e, "UCX", dtl_mode_env_len) == 0) { + dtl_mode = DYAD_DTL_UCX; + } else { + if (debug) { + fprintf (stderr, + "Invalid DTL mode provided through %s. \ + Defaulting to UCX\n", + DYAD_DTL_MODE_ENV); + } + dtl_mode = DYAD_DTL_DEFAULT; + } + } else { + dtl_mode = DYAD_DTL_DEFAULT; + } + if (debug) + fprintf (stderr, + "DYAD_CORE: retrieved configuration from environment. Now " + "initializing DYAD\n"); + return dyad_init (debug, + check, + shared_storage, + key_depth, + key_bins, + kvs_namespace, + prod_managed_path, + cons_managed_path, + dtl_mode, + ctx); +} + dyad_rc_t dyad_produce (dyad_ctx_t* ctx, const char* fname) { // If the context is not defined, then it is not valid. @@ -574,8 +703,7 @@ dyad_rc_t dyad_produce (dyad_ctx_t* ctx, const char* fname) } // If the producer-managed path is NULL or empty, then the context is not // valid for a producer operation. So, return DYAD_BADMANAGEDPATH - if (ctx->prod_managed_path == NULL - || strlen (ctx->prod_managed_path) == 0) { + if (ctx->prod_managed_path == NULL || strlen (ctx->prod_managed_path) == 0) { return DYAD_RC_BADMANAGEDPATH; } // If the context is valid, call dyad_commit to perform @@ -593,8 +721,7 @@ dyad_rc_t dyad_consume (dyad_ctx_t* ctx, const char* fname) } // If the consumer-managed path is NULL or empty, then the context is not // valid for a consumer operation. So, return DYAD_BADMANAGEDPATH - if (ctx->cons_managed_path == NULL - || strlen (ctx->cons_managed_path) == 0) { + if (ctx->cons_managed_path == NULL || strlen (ctx->cons_managed_path) == 0) { return DYAD_RC_BADMANAGEDPATH; } // Set reenter to false to avoid recursively performing @@ -621,7 +748,7 @@ dyad_rc_t dyad_consume (dyad_ctx_t* ctx, const char* fname) } // Call dyad_pull to fetch the data from the producer's // Flux broker - rc = dyad_pull (ctx, fname, resp); + rc = dyad_pull (ctx, resp); // Regardless if there was an error in dyad_pull, // free the KVS response object if (resp != NULL) { @@ -647,6 +774,7 @@ int dyad_finalize (dyad_ctx_t** ctx) if (ctx == NULL || *ctx == NULL) { return DYAD_RC_OK; } + dyad_dtl_finalize (&(*ctx)->dtl_handle); if ((*ctx)->h != NULL) { flux_close ((*ctx)->h); (*ctx)->h = NULL; @@ -669,10 +797,7 @@ int dyad_finalize (dyad_ctx_t** ctx) } #if DYAD_SYNC_DIR -#if DYAD_PERFFLOW -__attribute__((annotate("@critical_path()"))) -#endif -int dyad_sync_directory(dyad_ctx_t* ctx, const char* path) +int dyad_sync_directory (dyad_ctx_t* restrict ctx, const char* restrict path) { // Flush new directory entry https://lwn.net/Articles/457671/ char path_copy[PATH_MAX + 1]; int odir_fd = -1; diff --git a/src/core/dyad_core.h b/src/core/dyad_core.h index 21ecdbbf..a84a6033 100644 --- a/src/core/dyad_core.h +++ b/src/core/dyad_core.h @@ -1,9 +1,10 @@ #ifndef DYAD_CORE_DYAD_CORE_H #define DYAD_CORE_DYAD_CORE_H -// Includes #include "dyad_envs.h" #include "dyad_rc.h" +// Includes +#include "dyad_dtl.h" #include "dyad_flux_log.h" #ifdef __cplusplus @@ -22,7 +23,7 @@ *****************************************************************************/ // Now defined in src/utils/utils.h -//#define DYAD_PATH_DELIM "/" +// #define DYAD_PATH_DELIM "/" #ifdef __cplusplus extern "C" { @@ -32,29 +33,23 @@ extern "C" { * @struct dyad_ctx */ struct dyad_ctx { - flux_t* h; // the Flux handle for DYAD - bool debug; // if true, perform debug logging - bool check; // if true, perform some check logging - bool reenter; // if false, do not recursively enter DYAD - bool initialized; // if true, DYAD is initialized - bool shared_storage; // if true, the managed path is shared - bool sync_started; // TODO - unsigned int key_depth; // Depth of bins for the Flux KVS - unsigned int key_bins; // Number of bins for the Flux KVS - uint32_t rank; // Flux rank for DYAD - char* kvs_namespace; // Flux KVS namespace for DYAD - char* prod_managed_path; // producer path managed by DYAD - char* cons_managed_path; // consumer path managed by DYAD + flux_t* h; // the Flux handle for DYAD + struct dyad_dtl* dtl_handle; // Opaque handle to DTL info + bool debug; // if true, perform debug logging + bool check; // if true, perform some check logging + bool reenter; // if false, do not recursively enter DYAD + bool initialized; // if true, DYAD is initialized + bool shared_storage; // if true, the managed path is shared + unsigned int key_depth; // Depth of bins for the Flux KVS + unsigned int key_bins; // Number of bins for the Flux KVS + uint32_t rank; // Flux rank for DYAD + char* kvs_namespace; // Flux KVS namespace for DYAD + char* prod_managed_path; // producer path managed by DYAD + char* cons_managed_path; // consumer path managed by DYAD }; -extern const struct dyad_ctx dyad_ctx_default; +DYAD_DLL_EXPORTED extern const struct dyad_ctx dyad_ctx_default; typedef struct dyad_ctx dyad_ctx_t; -struct dyad_kvs_response { - char* fpath; - uint32_t owner_rank; -}; -typedef struct dyad_kvs_response dyad_kvs_response_t; - // Debug message #ifndef DPRINTF #define DPRINTF(curr_dyad_ctx, fmt, ...) \ @@ -91,15 +86,23 @@ typedef struct dyad_kvs_response dyad_kvs_response_t; * * @return An integer error code (values TBD) */ -dyad_rc_t dyad_init (bool debug, - bool check, - bool shared_storage, - unsigned int key_depth, - unsigned int key_bins, - const char* kvs_namespace, - const char* prod_managed_path, - const char* cons_managed_path, - dyad_ctx_t** ctx); +DYAD_DLL_EXPORTED dyad_rc_t dyad_init (bool debug, + bool check, + bool shared_storage, + unsigned int key_depth, + unsigned int key_bins, + const char* kvs_namespace, + const char* prod_managed_path, + const char* cons_managed_path, + dyad_dtl_mode_t dtl_mode, + dyad_ctx_t** ctx); + +/** + * @brief Intialize the DYAD context using environment variables + * @param[out] ctx the newly initialized context + * @return An error code + */ +DYAD_DLL_EXPORTED dyad_rc_t dyad_init_env (dyad_ctx_t** ctx); /** * @brief Wrapper function that performs all the common tasks needed @@ -109,11 +112,8 @@ dyad_rc_t dyad_init (bool debug, * * @return An integer error code (values TBD) */ -#if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) -#endif -dyad_rc_t -dyad_produce (dyad_ctx_t* ctx, const char* fname); +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_produce (dyad_ctx_t* ctx, + const char* fname); /** * @brief Wrapper function that performs all the common tasks needed @@ -123,11 +123,8 @@ dyad_produce (dyad_ctx_t* ctx, const char* fname); * * @return An integer error code (values TBD) */ -#if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) -#endif -dyad_rc_t -dyad_consume (dyad_ctx_t* ctx, const char* fname); +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume (dyad_ctx_t* ctx, + const char* fname); /** * @brief Finalizes the DYAD instance and deallocates the context @@ -135,10 +132,11 @@ dyad_consume (dyad_ctx_t* ctx, const char* fname); * * @return An integer error code (values TBD) */ -dyad_rc_t dyad_finalize (dyad_ctx_t** ctx); +DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize (dyad_ctx_t** ctx); #if DYAD_SYNC_DIR -int dyad_sync_directory (dyad_ctx_t* ctx, const char* path); +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED int dyad_sync_directory (dyad_ctx_t* ctx, + const char* path); #endif #ifdef __cplusplus diff --git a/src/core/dyad_core.sym b/src/core/dyad_core.sym deleted file mode 100644 index 68e72d86..00000000 --- a/src/core/dyad_core.sym +++ /dev/null @@ -1,6 +0,0 @@ -dyad_init -dyad_init_env -dyad_consume -dyad_produce -dyad_finalize -dyad_ctx_default diff --git a/src/core/dyad_envs.h b/src/core/dyad_envs.h index 422b37f1..86db96c3 100644 --- a/src/core/dyad_envs.h +++ b/src/core/dyad_envs.h @@ -4,6 +4,7 @@ #define DYAD_PATH_PRODUCER_ENV "DYAD_PATH_PRODUCER" #define DYAD_PATH_CONSUMER_ENV "DYAD_PATH_CONSUMER" #define DYAD_KVS_NAMESPACE_ENV "DYAD_KVS_NAMESPACE" +#define DYAD_DTL_MODE_ENV "DYAD_DTL_MODE" #define DYAD_SHARED_STORAGE_ENV "DYAD_SHARED_STORAGE" #define DYAD_KEY_DEPTH_ENV "DYAD_KEY_DEPTH" #define DYAD_KEY_BINS_ENV "DYAD_KEY_BINS" diff --git a/src/core/dyad_rc.h b/src/core/dyad_rc.h deleted file mode 100644 index c3ed2730..00000000 --- a/src/core/dyad_rc.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef DYAD_CORE_DYAD_RC_H -#define DYAD_CORE_DYAD_RC_H - -enum dyad_core_return_codes { - DYAD_RC_OK = 0, // Operation worked correctly - DYAD_RC_NOCTX = -1, // No DYAD Context found - DYAD_RC_FLUXFAIL = -2, // Some Flux function failed - DYAD_RC_BADCOMMIT = -3, // Flux KVS commit didn't work - DYAD_RC_BADLOOKUP = -4, // Flux KVS lookup didn't work - DYAD_RC_BADFETCH = -5, // Flux KVS commit didn't work - DYAD_RC_BADRESPONSE = -6, // Cannot create/populate a DYAD response - DYAD_RC_BADRPC = -7, // Flux RPC pack or get didn't work - DYAD_RC_BADFIO = -8, // File I/O failed - DYAD_RC_BADMANAGEDPATH = -9, // Cons or Prod Manged Path is bad -}; - -typedef enum dyad_core_return_codes dyad_rc_t; - -#define DYAD_IS_ERROR(code) ((code) < 0) - -#endif // DYAD_CORE_DYAD_RC_H diff --git a/src/dtl/Makefile.am b/src/dtl/Makefile.am new file mode 100644 index 00000000..287e8158 --- /dev/null +++ b/src/dtl/Makefile.am @@ -0,0 +1,25 @@ +noinst_LTLIBRARIES = libdyad_dtl.la +libdyad_dtl_la_SOURCES = \ + dyad_dtl_impl.c \ + dyad_dtl_impl.h \ + dyad_rc.h \ + flux_dtl.c \ + flux_dtl.h \ + ucx_dtl.c \ + ucx_dtl.h +libdyad_dtl_la_LIBADD = \ + $(top_builddir)/src/utils/libutils.la \ + $(top_builddir)/src/utils/libmurmur3.la \ + $(UCX_LIBS) \ + $(JANSSON_LIBS) \ + $(FLUX_CORE_LIBS) +libdyad_dtl_la_CFLAGS = \ + $(AM_CFLAGS) \ + -I$(top_srcdir)/src/utils \ + -I$(top_srcdir)/src/utils/base64 \ + $(UCX_CFLAGS) \ + $(JANSSON_CFLAGS) \ + $(FLUX_CORE_CLFAGS) \ + -fvisibility=hidden + +include_HEADERS = dyad_rc.h dyad_flux_log.h dyad_dtl.h \ No newline at end of file diff --git a/src/dtl/dyad_dtl.h b/src/dtl/dyad_dtl.h new file mode 100644 index 00000000..0ff6e544 --- /dev/null +++ b/src/dtl/dyad_dtl.h @@ -0,0 +1,19 @@ +#ifndef DYAD_DTL_DYAD_DTL_H +#define DYAD_DTL_DYAD_DTL_H + +#ifdef __cplusplus +extern "C" { +#endif + +enum dyad_dtl_mode { DYAD_DTL_UCX = 0, DYAD_DTL_FLUX_RPC = 1, DYAD_DTL_END = 2 }; +typedef enum dyad_dtl_mode dyad_dtl_mode_t; + +static const dyad_dtl_mode_t DYAD_DTL_DEFAULT = DYAD_DTL_FLUX_RPC; + +struct dyad_dtl; + +#ifdef __cplusplus +} +#endif + +#endif /* DYAD_DTL_DYAD_DTL_H */ \ No newline at end of file diff --git a/src/dtl/dyad_dtl_impl.c b/src/dtl/dyad_dtl_impl.c new file mode 100644 index 00000000..dd002909 --- /dev/null +++ b/src/dtl/dyad_dtl_impl.c @@ -0,0 +1,62 @@ +#include "dyad_dtl_impl.h" + +#include "flux_dtl.h" +#include "ucx_dtl.h" + +dyad_rc_t dyad_dtl_init (dyad_dtl_t **dtl_handle, + dyad_dtl_mode_t mode, + flux_t *h, + bool debug) +{ + dyad_rc_t rc = DYAD_RC_OK; + *dtl_handle = malloc (sizeof (struct dyad_dtl)); + if (*dtl_handle == NULL) { + return DYAD_RC_SYSFAIL; + } + (*dtl_handle)->mode = mode; + if (mode == DYAD_DTL_UCX) { + rc = dyad_dtl_ucx_init (*dtl_handle, mode, h, debug); + if (DYAD_IS_ERROR (rc)) { + return rc; + } + return DYAD_RC_OK; + } else if (mode == DYAD_DTL_FLUX_RPC) { + rc = dyad_dtl_flux_init (*dtl_handle, mode, h, debug); + if (DYAD_IS_ERROR (rc)) { + return rc; + } + return DYAD_RC_OK; + } + return DYAD_RC_BADDTLMODE; +} + +dyad_rc_t dyad_dtl_finalize (dyad_dtl_t **dtl_handle) +{ + dyad_rc_t rc = DYAD_RC_OK; + if (dtl_handle == NULL || *dtl_handle == NULL) + // We should only reach this line if the user has passed + // in an already-finalized DTL handle. In that case, + // this function should be treated as a no-op, and we + // should return DYAD_RC_OK to indicate no error has occured + return DYAD_RC_OK; + if ((*dtl_handle)->mode == DYAD_DTL_UCX) { + if ((*dtl_handle)->private.ucx_dtl_handle != NULL) { + rc = dyad_dtl_ucx_finalize (dtl_handle); + if (DYAD_IS_ERROR (rc)) { + return rc; + } + } + } else if ((*dtl_handle)->mode == DYAD_DTL_FLUX_RPC) { + if ((*dtl_handle)->private.flux_dtl_handle != NULL) { + rc = dyad_dtl_flux_finalize (dtl_handle); + if (DYAD_IS_ERROR (rc)) { + return rc; + } + } + } else { + return DYAD_RC_BADDTLMODE; + } + free (*dtl_handle); + *dtl_handle = NULL; + return DYAD_RC_OK; +} diff --git a/src/dtl/dyad_dtl_impl.h b/src/dtl/dyad_dtl_impl.h new file mode 100644 index 00000000..5ad367c9 --- /dev/null +++ b/src/dtl/dyad_dtl_impl.h @@ -0,0 +1,72 @@ +#ifndef DYAD_DTL_DYAD_DTL_IMPL_H +#define DYAD_DTL_DYAD_DTL_IMPL_H + +#include +#include + +#include "dyad_dtl.h" +#include "dyad_flux_log.h" +#include "dyad_rc.h" + +#ifdef __cplusplus +#include + +extern "C" { +#else +#include +#endif + +#define DYAD_DTL_RPC_NAME "dyad.fetch" + +// Forward declarations of DTL contexts for the underlying implementations +struct dyad_dtl_ucx; +struct dyad_dtl_flux; + +// Union type to store the underlying DTL contexts +union dyad_dtl_private { + struct dyad_dtl_ucx* ucx_dtl_handle; + struct dyad_dtl_flux* flux_dtl_handle; +}; +typedef union dyad_dtl_private dyad_dtl_private_t; + +enum dyad_dtl_comm_mode { + DYAD_COMM_NONE = 0, // Sanity check value for when + // connection isn't established + DYAD_COMM_RECV = 1, // DTL connection will only receive data + DYAD_COMM_SEND = 2, // DTL connection will only send data + DYAD_COMM_END = 3 +}; +typedef enum dyad_dtl_comm_mode dyad_dtl_comm_mode_t; + +struct dyad_dtl { + dyad_dtl_private_t private; + dyad_dtl_mode_t mode; + dyad_rc_t (*rpc_pack) (struct dyad_dtl* restrict self, + const char* restrict upath, + uint32_t producer_rank, + json_t** restrict packed_obj); + dyad_rc_t (*rpc_unpack) (struct dyad_dtl* self, + const flux_msg_t* packed_obj, + char** upath); + dyad_rc_t (*rpc_respond) (struct dyad_dtl* self, const flux_msg_t* orig_msg); + dyad_rc_t (*rpc_recv_response) (struct dyad_dtl* self, flux_future_t* f); + dyad_rc_t (*establish_connection) (struct dyad_dtl* self, + dyad_dtl_comm_mode_t comm_mode); + dyad_rc_t (*send) (struct dyad_dtl* self, void* buf, size_t buflen); + dyad_rc_t (*recv) (struct dyad_dtl* self, void** buf, size_t* buflen); + dyad_rc_t (*close_connection) (struct dyad_dtl* self); +}; +typedef struct dyad_dtl dyad_dtl_t; + +dyad_rc_t dyad_dtl_init (dyad_dtl_t** dtl_handle, + dyad_dtl_mode_t mode, + flux_t* h, + bool debug); + +dyad_rc_t dyad_dtl_finalize (dyad_dtl_t** dtl_handle); + +#ifdef __cplusplus +} +#endif + +#endif /* DYAD_DTL_DYAD_DTL_IMPL_H */ diff --git a/src/core/dyad_flux_log.h b/src/dtl/dyad_flux_log.h similarity index 73% rename from src/core/dyad_flux_log.h rename to src/dtl/dyad_flux_log.h index 45dd9c26..d6376c46 100644 --- a/src/core/dyad_flux_log.h +++ b/src/dtl/dyad_flux_log.h @@ -1,8 +1,12 @@ -#ifndef DYAD_CORE_DYAD_FLUX_LOG_H -#define DYAD_CORE_DYAD_FLUX_LOG_H +#ifndef DYAD_DTL_DYAD_FLUX_LOG_H +#define DYAD_DTL_DYAD_FLUX_LOG_H #include +#ifdef __cplusplus +extern "C" { +#endif + #if !defined(DYAD_LOGGING_ON) || (DYAD_LOGGING_ON == 0) #define DYAD_LOG_INFO(dyad_ctx, ...) \ do { \ @@ -17,11 +21,14 @@ do { \ } while (0) #else -#define DYAD_LOG_INFO(dyad_ctx, ...) \ - flux_log (dyad_ctx->h, LOG_INFO, __VA_ARGS__) +#define DYAD_LOG_INFO(dyad_ctx, ...) flux_log (dyad_ctx->h, LOG_INFO, __VA_ARGS__) #define DYAD_LOG_ERR(dyad_ctx, ...) flux_log_error (dyad_ctx->h, __VA_ARGS__) #define FLUX_LOG_INFO(flux_ctx, ...) flux_log (flux_ctx, LOG_INFO, __VA_ARGS__) #define FLUX_LOG_ERR(flux_ctx, ...) flux_log_error (flux_ctx, __VA_ARGS__) #endif -#endif /* DYAD_CORE_DYAD_FLUX_LOG_H */ +#ifdef __cplusplus +} +#endif + +#endif /* DYAD_DTL_DYAD_FLUX_LOG_H */ diff --git a/src/dtl/dyad_rc.h b/src/dtl/dyad_rc.h new file mode 100644 index 00000000..5ffccffc --- /dev/null +++ b/src/dtl/dyad_rc.h @@ -0,0 +1,56 @@ +#ifndef DYAD_DTL_DYAD_RC_H +#define DYAD_DTL_DYAD_RC_H + +#if BUILDING_DYAD +#define DYAD_DLL_EXPORTED __attribute__ ((__visibility__ ("default"))) +#else +#define DYAD_DLL_EXPORTED +#endif + +#if DYAD_PERFFLOW +#define DYAD_PFA_ANNOTATE __attribute__ ((annotate ("@critical_path()"))) +#else +#define DYAD_PFA_ANNOTATE +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +enum dyad_core_return_codes { + DYAD_RC_OK = 0, // Operation worked correctly + DYAD_RC_SYSFAIL = -1, // Some sys call or C standard + // library call failed + DYAD_RC_NOCTX = -2, // No DYAD Context found + DYAD_RC_FLUXFAIL = -3, // Some Flux function failed + DYAD_RC_BADCOMMIT = -4, // Flux KVS commit didn't work + DYAD_RC_BADLOOKUP = -5, // Flux KVS lookup didn't work + DYAD_RC_BADFETCH = -6, // Flux KVS commit didn't work + DYAD_RC_BADRESPONSE = -7, // Cannot create/populate a DYAD response + DYAD_RC_BADRPC = -8, // Flux RPC pack or get didn't work + DYAD_RC_BADFIO = -9, // File I/O failed + DYAD_RC_BADMANAGEDPATH = -10, // Cons or Prod Manged Path is bad + DYAD_RC_BADDTLMODE = -11, // Invalid DYAD DTL mode provided + DYAD_RC_BADPACK = -12, // JSON packing failed + DYAD_RC_BADUNPACK = -13, // JSON unpacking failed + DYAD_RC_UCXINIT_FAIL = -14, // UCX initialization failed + DYAD_RC_UCXWAIT_FAIL = -15, // UCX wait (either custom or + // 'ucp_worker_wait') failed + DYAD_RC_UCXCOMM_FAIL = -16, // UCX communication routine failed + DYAD_RC_RPC_FINISHED = -17, // The Flux RPC responded with ENODATA (i.e., + // end of stream) sooner than expected + DYAD_RC_BAD_B64DECODE = -18, // Decoding of data w/ base64 failed + DYAD_RC_BAD_COMM_MODE = -19, // Invalid comm mode provided to DTL +}; + +typedef enum dyad_core_return_codes dyad_rc_t; + +#define DYAD_IS_ERROR(code) ((code) < 0) + +#define FLUX_IS_ERROR(code) ((code) < 0) + +#ifdef __cplusplus +} +#endif + +#endif // DYAD_DTL_DYAD_RC_H diff --git a/src/dtl/flux_dtl.c b/src/dtl/flux_dtl.c new file mode 100644 index 00000000..7d34b039 --- /dev/null +++ b/src/dtl/flux_dtl.c @@ -0,0 +1,155 @@ +#include "flux_dtl.h" + +dyad_rc_t dyad_dtl_flux_init (dyad_dtl_t* self, + dyad_dtl_mode_t mode, + flux_t* h, + bool debug) +{ + self->private.flux_dtl_handle = malloc (sizeof (struct dyad_dtl_flux)); + if (self->private.flux_dtl_handle == NULL) { + FLUX_LOG_ERR (h, "Cannot allocate the Flux DTL handle\n"); + return DYAD_RC_SYSFAIL; + } + self->private.flux_dtl_handle->h = h; + self->private.flux_dtl_handle->debug = debug; + self->private.flux_dtl_handle->f = NULL; + self->private.flux_dtl_handle->msg = NULL; + + self->rpc_pack = dyad_dtl_flux_rpc_pack; + self->rpc_unpack = dyad_dtl_flux_rpc_unpack; + self->rpc_respond = dyad_dtl_flux_rpc_respond; + self->rpc_recv_response = dyad_dtl_flux_rpc_recv_response; + self->establish_connection = dyad_dtl_flux_establish_connection; + self->send = dyad_dtl_flux_send; + self->recv = dyad_dtl_flux_recv; + self->close_connection = dyad_dtl_flux_close_connection; + + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_flux_rpc_pack (dyad_dtl_t* restrict self, + const char* restrict upath, + uint32_t producer_rank, + json_t** restrict packed_obj) +{ + dyad_dtl_flux_t* dtl_handle = self->private.flux_dtl_handle; + *packed_obj = json_pack ("{s:s}", "upath", upath); + if (*packed_obj == NULL) { + FLUX_LOG_ERR (dtl_handle->h, "Could not pack upath for Flux DTL\n"); + return DYAD_RC_BADPACK; + } + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_flux_rpc_unpack (dyad_dtl_t* self, const flux_msg_t* msg, char** upath) +{ + int rc = 0; + rc = flux_request_unpack (msg, NULL, "{s:s}", "upath", upath); + if (FLUX_IS_ERROR (rc)) { + FLUX_LOG_ERR (self->private.flux_dtl_handle->h, + "Could not unpack Flux message from consumer\n"); + // TODO create new RC for this + return DYAD_RC_BADUNPACK; + } + self->private.flux_dtl_handle->msg = msg; + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_flux_rpc_respond (dyad_dtl_t* self, const flux_msg_t* orig_msg) +{ + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_flux_rpc_recv_response (dyad_dtl_t* self, flux_future_t* f) +{ + self->private.flux_dtl_handle->f = f; + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_flux_establish_connection (dyad_dtl_t* self, + dyad_dtl_comm_mode_t comm_mode) +{ + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_flux_send (dyad_dtl_t* self, void* buf, size_t buflen) +{ + int rc = 0; + FLUX_LOG_INFO (self->private.flux_dtl_handle->h, + "Send data to consumer using a Flux RPC response"); + rc = flux_respond_raw (self->private.flux_dtl_handle->h, + self->private.flux_dtl_handle->msg, + buf, + (int)buflen); + if (FLUX_IS_ERROR (rc)) { + FLUX_LOG_ERR (self->private.flux_dtl_handle->h, + "Could not send Flux RPC response containing file " + "contents\n"); + return DYAD_RC_FLUXFAIL; + } + if (self->private.flux_dtl_handle->debug) { + FLUX_LOG_INFO (self->private.flux_dtl_handle->h, + "Successfully sent file contents to consumer\n"); + } + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_flux_recv (dyad_dtl_t* self, void** buf, size_t* buflen) +{ + int rc = 0; + dyad_rc_t dyad_rc = DYAD_RC_OK; + errno = 0; + dyad_dtl_flux_t* dtl_handle = self->private.flux_dtl_handle; + FLUX_LOG_INFO (dtl_handle->h, "Get file contents from module using Flux RPC\n"); + if (dtl_handle->f == NULL) { + FLUX_LOG_ERR (dtl_handle->h, "Cannot get data using RPC without a Flux future\n"); + // TODO create new RC for this + return DYAD_RC_FLUXFAIL; + } + void* tmp_buf; + size_t tmp_buflen; + rc = flux_rpc_get_raw (dtl_handle->f, (const void**)&tmp_buf, (int*)&tmp_buflen); + if (FLUX_IS_ERROR (rc)) { + FLUX_LOG_ERR (dtl_handle->h, "Could not get file data from Flux RPC"); + if (errno == ENODATA) + dyad_rc = DYAD_RC_RPC_FINISHED; + else + dyad_rc = DYAD_RC_BADRPC; + goto finish_recv; + } + *buflen = tmp_buflen; + if (*buf == NULL) { + *buf = malloc (tmp_buflen); + if (*buf == NULL) { + FLUX_LOG_ERR (dtl_handle->h, "Could not allocate space for file buffer"); + dyad_rc = DYAD_RC_SYSFAIL; + goto finish_recv; + } + } + memcpy (*buf, tmp_buf, tmp_buflen); + dyad_rc = DYAD_RC_OK; +finish_recv: + flux_future_reset (dtl_handle->f); + return dyad_rc; +} + +dyad_rc_t dyad_dtl_flux_close_connection (dyad_dtl_t* self) +{ + if (self->private.flux_dtl_handle->f != NULL) + self->private.flux_dtl_handle->f = NULL; + if (self->private.flux_dtl_handle->msg != NULL) + self->private.flux_dtl_handle->msg = NULL; + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_flux_finalize (dyad_dtl_t** self) +{ + if (self == NULL || *self == NULL) + return DYAD_RC_OK; + (*self)->private.flux_dtl_handle->h = NULL; + (*self)->private.flux_dtl_handle->f = NULL; + (*self)->private.flux_dtl_handle->msg = NULL; + free ((*self)->private.flux_dtl_handle); + (*self)->private.flux_dtl_handle = NULL; + return DYAD_RC_OK; +} \ No newline at end of file diff --git a/src/dtl/flux_dtl.h b/src/dtl/flux_dtl.h new file mode 100644 index 00000000..f58580b6 --- /dev/null +++ b/src/dtl/flux_dtl.h @@ -0,0 +1,46 @@ +#ifndef DYAD_DTL_FLUX_H +#define DYAD_DTL_FLUX_H + +#include + +#include "dyad_dtl_impl.h" + +struct dyad_dtl_flux { + flux_t* h; + bool debug; + flux_future_t* f; + flux_msg_t* msg; +}; + +typedef struct dyad_dtl_flux dyad_dtl_flux_t; + +dyad_rc_t dyad_dtl_flux_init (dyad_dtl_t* self, + dyad_dtl_mode_t mode, + flux_t* h, + bool debug); + +dyad_rc_t dyad_dtl_flux_rpc_pack (dyad_dtl_t* restrict self, + const char* restrict upath, + uint32_t producer_rank, + json_t** restrict packed_obj); + +dyad_rc_t dyad_dtl_flux_rpc_unpack (dyad_dtl_t* self, + const flux_msg_t* msg, + char** upath); + +dyad_rc_t dyad_dtl_flux_rpc_respond (dyad_dtl_t* self, const flux_msg_t* orig_msg); + +dyad_rc_t dyad_dtl_flux_rpc_recv_response (dyad_dtl_t* self, flux_future_t* f); + +dyad_rc_t dyad_dtl_flux_establish_connection (dyad_dtl_t* self, + dyad_dtl_comm_mode_t comm_mode); + +dyad_rc_t dyad_dtl_flux_send (dyad_dtl_t* self, void* buf, size_t buflen); + +dyad_rc_t dyad_dtl_flux_recv (dyad_dtl_t* self, void** buf, size_t* buflen); + +dyad_rc_t dyad_dtl_flux_close_connection (dyad_dtl_t* self); + +dyad_rc_t dyad_dtl_flux_finalize (dyad_dtl_t** self); + +#endif /* DYAD_DTL_FLUX_H */ \ No newline at end of file diff --git a/src/dtl/ucx_dtl.c b/src/dtl/ucx_dtl.c new file mode 100644 index 00000000..634f9da2 --- /dev/null +++ b/src/dtl/ucx_dtl.c @@ -0,0 +1,687 @@ +#include "ucx_dtl.h" + +#include +#include +#include +#include + +#include "base64.h" + +extern const base64_maps_t base64_maps_rfc4648; + +// Tag mask for UCX Tag send/recv +#define DYAD_UCX_TAG_MASK UINT64_MAX + +// Macro function used to simplify checking the status +// of UCX operations +#define UCX_STATUS_FAIL(status) (status != UCS_OK) + +// Define a request struct to be used in handling +// async UCX operations +struct ucx_request { + int completed; +}; +typedef struct ucx_request dyad_ucx_request_t; + +// Define a function that UCX will use to allocate and +// initialize our request struct +static void dyad_ucx_request_init (void* request) +{ + dyad_ucx_request_t* real_request = NULL; + real_request = (dyad_ucx_request_t*)request; + real_request->completed = 0; +} + +// Define a function that ucp_tag_msg_recv_nbx will use +// as a callback to signal the completion of the async receive +#if UCP_API_VERSION >= UCP_VERSION(1, 10) +static void dyad_recv_callback (void* request, + ucs_status_t status, + const ucp_tag_recv_info_t* tag_info, + void* user_data) +#else +static void dyad_recv_callback (void* request, + ucs_status_t status, + ucp_tag_recv_info_t* tag_info) +#endif +{ + dyad_ucx_request_t* real_request = NULL; + real_request = (dyad_ucx_request_t*)request; + real_request->completed = 1; +} + +#if UCP_API_VERSION >= UCP_VERSION(1, 10) +static void dyad_send_callback (void* req, ucs_status_t status, void* ctx) +#else +static void dyad_send_callback (void* req, ucs_status_t status) +#endif +{ + dyad_ucx_request_t* real_req = (dyad_ucx_request_t*)req; + real_req->completed = 1; +} + +void dyad_ucx_ep_err_handler (void* arg, ucp_ep_h ep, ucs_status_t status) +{ + flux_t* h = (flux_t*)arg; + FLUX_LOG_ERR (h, "An error occured on the UCP endpoint (status = %d)\n", status); +} + +// Simple function used to wait on the async receive +static ucs_status_t dyad_ucx_request_wait (dyad_dtl_ucx_t* dtl_handle, + dyad_ucx_request_t* request) +{ + ucs_status_t final_request_status = UCS_OK; + // If 'request' is actually a request handle, this means the communication + // operation is scheduled, but not yet completed. + if (UCS_PTR_IS_PTR (request)) { + // Spin lock until the request is completed + // The spin lock shouldn't be costly (performance-wise) + // because the wait should always come directly after other UCX calls + // that minimize the size of the worker's event queue. + // In other words, prior UCX calls should mean that this loop only runs + // a couple of times at most. + do { + ucp_worker_progress (dtl_handle->ucx_worker); + // usleep(100); + // Get the final status of the communication operation + final_request_status = ucp_request_check_status (request); + } while (final_request_status == UCS_INPROGRESS); + // Free and deallocate the request object + ucp_request_free (request); + return final_request_status; + } + // If 'request' is actually a UCX error, this means the communication + // operation immediately failed. In that case, we simply grab the + // 'ucs_status_t' object for the error. + else if (UCS_PTR_IS_ERR (request)) { + return UCS_PTR_STATUS (request); + } + // If 'request' is neither a request handle nor an error, then + // the communication operation immediately completed successfully. + // So, we simply set the status to UCS_OK + return UCS_OK; +} + +static inline dyad_rc_t dyad_dtl_ucx_finalize_impl (dyad_dtl_ucx_t** dtl_handle) +{ +} + +dyad_rc_t dyad_dtl_ucx_init (dyad_dtl_t* self, + dyad_dtl_mode_t mode, + flux_t* h, + bool debug) +{ + ucp_params_t ucx_params; + ucp_worker_params_t worker_params; + ucp_config_t* config; + ucs_status_t status; + ucp_worker_attr_t worker_attrs; + dyad_dtl_ucx_t* dtl_handle = NULL; + + self->private.ucx_dtl_handle = malloc (sizeof (struct dyad_dtl_ucx)); + if (self->private.ucx_dtl_handle == NULL) { + FLUX_LOG_ERR (h, "Could not allocate UCX DTL context\n"); + return DYAD_RC_SYSFAIL; + } + dtl_handle = self->private.ucx_dtl_handle; + // Allocation/Freeing of the Flux handle should be + // handled by the DYAD context + dtl_handle->h = h; + dtl_handle->debug = debug; + dtl_handle->ucx_ctx = NULL; + dtl_handle->ucx_worker = NULL; + dtl_handle->ep = NULL; + dtl_handle->curr_comm_mode = DYAD_COMM_NONE; + dtl_handle->consumer_address = NULL; + dtl_handle->addr_len = 0; + dtl_handle->comm_tag = 0; + + // Read the UCX configuration + FLUX_LOG_INFO (dtl_handle->h, "Reading UCP config\n"); + status = ucp_config_read (NULL, NULL, &config); + if (UCX_STATUS_FAIL (status)) { + FLUX_LOG_ERR (dtl_handle->h, "Could not read the UCX config\n"); + goto error; + } + + // Define the settings, parameters, features, etc. + // for the UCX context. UCX will use this info internally + // when creating workers, endpoints, etc. + // + // The settings enabled are: + // * Tag-matching send/recv + // * Remote Memory Access communication + // * Auto initialization of request objects + // * Worker sleep, wakeup, poll, etc. features + ucx_params.field_mask = UCP_PARAM_FIELD_FEATURES | UCP_PARAM_FIELD_REQUEST_SIZE + | UCP_PARAM_FIELD_REQUEST_INIT; + ucx_params.features = UCP_FEATURE_TAG | + // UCP_FEATURE_RMA | + UCP_FEATURE_WAKEUP; + ucx_params.request_size = sizeof (struct ucx_request); + ucx_params.request_init = dyad_ucx_request_init; + + // Initialize UCX + FLUX_LOG_INFO (dtl_handle->h, "Initializing UCP\n"); + status = ucp_init (&ucx_params, config, &dtl_handle->ucx_ctx); + + // If in debug mode, print the configuration of UCX to stderr + if (debug) { + ucp_config_print (config, stderr, "UCX Configuration", UCS_CONFIG_PRINT_CONFIG); + } + // Release the config + ucp_config_release (config); + // Log an error if UCX initialization failed + if (UCX_STATUS_FAIL (status)) { + FLUX_LOG_ERR (h, "ucp_init failed (status = %d)\n", status); + goto error; + } + + // Define the settings for the UCX worker (i.e., progress engine) + // + // The settings enabled are: + // * Single-threaded mode (TODO look into multi-threading support) + // * Restricting wakeup events to only include Tag-matching recv events + worker_params.field_mask = + UCP_WORKER_PARAM_FIELD_THREAD_MODE | UCP_WORKER_PARAM_FIELD_EVENTS; + // TODO look into multi-threading support + worker_params.thread_mode = UCS_THREAD_MODE_SINGLE; + worker_params.events = UCP_WAKEUP_TAG_RECV; + + // Create the worker and log an error if that fails + FLUX_LOG_INFO (dtl_handle->h, "Creating UCP worker\n"); + status = ucp_worker_create (dtl_handle->ucx_ctx, + &worker_params, + &(dtl_handle->ucx_worker)); + if (UCX_STATUS_FAIL (status)) { + FLUX_LOG_ERR (dtl_handle->h, "ucp_worker_create failed (status = %d)!\n", status); + goto error; + } + + // Query the worker for its address + worker_attrs.field_mask = UCP_WORKER_ATTR_FIELD_ADDRESS; + FLUX_LOG_INFO (dtl_handle->h, "Get address of UCP worker\n"); + status = ucp_worker_query (dtl_handle->ucx_worker, &worker_attrs); + if (UCX_STATUS_FAIL (status)) { + FLUX_LOG_ERR (h, "Cannot get UCX worker address (status = %d)!\n", status); + goto error; + } + dtl_handle->consumer_address = worker_attrs.address; + dtl_handle->addr_len = worker_attrs.address_length; + + self->rpc_pack = dyad_dtl_ucx_rpc_pack; + self->rpc_unpack = dyad_dtl_ucx_rpc_unpack; + self->rpc_respond = dyad_dtl_ucx_rpc_respond; + self->rpc_recv_response = dyad_dtl_ucx_rpc_recv_response; + self->establish_connection = dyad_dtl_ucx_establish_connection; + self->send = dyad_dtl_ucx_send; + self->recv = dyad_dtl_ucx_recv; + self->close_connection = dyad_dtl_ucx_close_connection; + + return DYAD_RC_OK; + +error:; + // If an error occured, finalize the DTL handle and + // return a failing error code + dyad_dtl_ucx_finalize (&self); + return DYAD_RC_UCXINIT_FAIL; +} + +dyad_rc_t dyad_dtl_ucx_rpc_pack (dyad_dtl_t* restrict self, + const char* restrict upath, + uint32_t producer_rank, + json_t** restrict packed_obj) +{ + size_t enc_len = 0; + char* enc_buf = NULL; + ssize_t enc_size = 0; + dyad_dtl_ucx_t* dtl_handle = self->private.ucx_dtl_handle; + if (dtl_handle->consumer_address == NULL) { + // TODO log error + return DYAD_RC_BADPACK; + } + FLUX_LOG_INFO (dtl_handle->h, "Encode UCP address using base64\n"); + enc_len = base64_encoded_length (dtl_handle->addr_len); + // Add 1 to encoded length because the encoded buffer will be + // packed as if it is a string + enc_buf = malloc (enc_len + 1); + if (enc_buf == NULL) { + FLUX_LOG_ERR (dtl_handle->h, "Could not allocate buffer for packed address\n"); + return DYAD_RC_SYSFAIL; + } + // consumer_address is casted to const char* to avoid warnings + // This is valid because it is a pointer to an opaque struct, + // so the cast can be treated like a void*->char* cast. + enc_size = base64_encode_using_maps (&base64_maps_rfc4648, + enc_buf, + enc_len + 1, + (const char*)dtl_handle->consumer_address, + dtl_handle->addr_len); + if (enc_size < 0) { + // TODO log error + free (enc_buf); + return DYAD_RC_BADPACK; + } + FLUX_LOG_INFO (dtl_handle->h, "Creating UCP tag for tag matching\n"); + // Because we're using tag-matching send/recv for communication, + // there's no need to do any real connection establishment here. + // Instead, we use this function to create the tag that will be + // used for the upcoming communication. + uint32_t consumer_rank = 0; + if (flux_get_rank (dtl_handle->h, &consumer_rank) < 0) { + FLUX_LOG_ERR (dtl_handle->h, "Cannot get consumer rank\n"); + return DYAD_RC_FLUXFAIL; + } + // The tag is a 64 bit unsigned integer consisting of the + // 32-bit rank of the producer followed by the 32-bit rank + // of the consumer + dtl_handle->comm_tag = ((uint64_t)producer_rank << 32) | (uint64_t)consumer_rank; + // Use Jansson to pack the tag and UCX address into + // the payload to be sent via RPC to the producer plugin + FLUX_LOG_INFO (dtl_handle->h, "Packing RPC payload for UCX DTL\n"); + *packed_obj = json_pack ("{s:s, s:i, s:i, s:s%}", + "upath", + upath, + "tag_prod", + (int)producer_rank, + "tag_cons", + (int)consumer_rank, + "ucx_addr", + enc_buf, + enc_len); + free (enc_buf); + // If the packing failed, log an error + if (*packed_obj == NULL) { + FLUX_LOG_ERR (dtl_handle->h, "Could not pack upath and UCX address for RPC\n"); + return DYAD_RC_BADPACK; + } + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_ucx_rpc_unpack (dyad_dtl_t* self, const flux_msg_t* msg, char** upath) +{ + char* enc_addr = NULL; + size_t enc_addr_len = 0; + int errcode = 0; + uint32_t tag_prod = 0; + uint32_t tag_cons = 0; + ssize_t decoded_len = 0; + dyad_dtl_ucx_t* dtl_handle = self->private.ucx_dtl_handle; + FLUX_LOG_INFO (dtl_handle->h, "Unpacking RPC payload\n"); + errcode = flux_request_unpack (msg, + NULL, + "{s:s, s:i, s:i, s:s%}", + "upath", + upath, + "tag_prod", + &tag_prod, + "tag_cons", + &tag_cons, + "ucx_addr", + &enc_addr, + &enc_addr_len); + if (errcode < 0) { + FLUX_LOG_ERR (dtl_handle->h, "Could not unpack Flux message from consumer!\n"); + return DYAD_RC_BADUNPACK; + } + dtl_handle->comm_tag = ((uint64_t)tag_prod << 32) | (uint64_t)tag_cons; + FLUX_LOG_INFO (dtl_handle->h, "Obtained upath from RPC payload: %s\n", upath); + FLUX_LOG_INFO (dtl_handle->h, + "Obtained UCP tag from RPC payload: %lu\n", + dtl_handle->comm_tag); + FLUX_LOG_INFO (dtl_handle->h, "Decoding consumer UCP address using base64\n"); + dtl_handle->addr_len = base64_decoded_length (enc_addr_len); + dtl_handle->consumer_address = (ucp_address_t*)malloc (dtl_handle->addr_len); + if (dtl_handle->consumer_address == NULL) { + FLUX_LOG_ERR (dtl_handle->h, "Could not allocate memory for consumer address"); + return DYAD_RC_SYSFAIL; + } + decoded_len = base64_decode_using_maps (&base64_maps_rfc4648, + (char*)dtl_handle->consumer_address, + dtl_handle->addr_len, + enc_addr, + enc_addr_len); + if (decoded_len < 0) { + // TODO log error + free (dtl_handle->consumer_address); + dtl_handle->consumer_address = NULL; + dtl_handle->addr_len = 0; + return DYAD_RC_BAD_B64DECODE; + } + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_ucx_rpc_respond (dyad_dtl_t* self, const flux_msg_t* orig_msg) +{ + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_ucx_rpc_recv_response (dyad_dtl_t* self, flux_future_t* f) +{ + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_ucx_establish_connection (dyad_dtl_t* self, + dyad_dtl_comm_mode_t comm_mode) +{ + ucp_ep_params_t params; + ucs_status_t status = UCS_OK; + dyad_dtl_ucx_t* dtl_handle = self->private.ucx_dtl_handle; + dtl_handle->curr_comm_mode = comm_mode; + if (comm_mode == DYAD_COMM_SEND) { + params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS + | UCP_EP_PARAM_FIELD_ERR_HANDLING_MODE + | UCP_EP_PARAM_FIELD_ERR_HANDLER; + params.address = dtl_handle->consumer_address; + params.err_mode = UCP_ERR_HANDLING_MODE_PEER; + params.err_handler.cb = dyad_ucx_ep_err_handler; + params.err_handler.arg = (void*)dtl_handle->h; + FLUX_LOG_INFO (dtl_handle->h, + "Create UCP endpoint for communication with consumer\n"); + status = ucp_ep_create (dtl_handle->ucx_worker, ¶ms, &dtl_handle->ep); + if (status != UCS_OK) { + FLUX_LOG_ERR (dtl_handle->h, + "ucp_ep_create failed with status %d\n", + (int)status); + return DYAD_RC_UCXCOMM_FAIL; + } + if (dtl_handle->debug) { + ucp_ep_print_info (dtl_handle->ep, stderr); + } + return DYAD_RC_OK; + } else if (comm_mode == DYAD_COMM_RECV) { + FLUX_LOG_INFO (dtl_handle->h, + "No explicit connection establishment needed for UCX " + "receiver\n"); + return DYAD_RC_OK; + } else { + FLUX_LOG_ERR (dtl_handle->h, "Invalid communication mode: %d\n", comm_mode); + // TODO create new RC for this + return DYAD_RC_BAD_COMM_MODE; + } +} + +dyad_rc_t dyad_dtl_ucx_send (dyad_dtl_t* self, void* buf, size_t buflen) +{ + ucs_status_ptr_t stat_ptr; + ucs_status_t status = UCS_OK; + dyad_ucx_request_t* req = NULL; + dyad_dtl_ucx_t* dtl_handle = self->private.ucx_dtl_handle; + if (dtl_handle->ep == NULL) { + FLUX_LOG_INFO (dtl_handle->h, + "UCP endpoint was not created prior to invoking " + "send!\n"); + return DYAD_RC_UCXCOMM_FAIL; + } + // ucp_tag_send_sync_nbx is the prefered version of this send since UCX 1.9 + // However, some systems (e.g., Lassen) may have an older verison + // This conditional compilation will use ucp_tag_send_sync_nbx if using + // UCX 1.9+, and it will use the deprecated ucp_tag_send_sync_nb if using + // UCX < 1.9. +#if UCP_API_VERSION >= UCP_VERSION(1, 10) + ucp_request_param_t params; + params.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK; + params.cb.send = dyad_send_callback; + FLUX_LOG_INFO (dtl_handle->h, "Sending data to consumer with ucp_tag_send_nbx\n"); + stat_ptr = + ucp_tag_send_nbx (dtl_handle->ep, buf, buflen, dtl_handle->comm_tag, ¶ms); +#else + FLUX_LOG_INFO (dtl_handle->h, + "Sending %lu bytes of data to consumer with " + "ucp_tag_send_nb\n", + buflen); + stat_ptr = ucp_tag_send_nb (dtl_handle->ep, + buf, + buflen, + UCP_DATATYPE_CONTIG, + dtl_handle->comm_tag, + dyad_send_callback); +#endif + FLUX_LOG_INFO (dtl_handle->h, "Processing UCP send request\n"); + status = dyad_ucx_request_wait (dtl_handle, stat_ptr); + if (status != UCS_OK) { + FLUX_LOG_ERR (dtl_handle->h, "UCP Tag Send failed (status = %d)!\n", (int)status); + return DYAD_RC_UCXCOMM_FAIL; + } + FLUX_LOG_INFO (dtl_handle->h, "Data send with UCP succeeded\n"); + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_ucx_recv (dyad_dtl_t* self, void** buf, size_t* buflen) +{ + ucs_status_t status = UCS_OK; + ucp_tag_message_h msg = NULL; + ucp_tag_recv_info_t msg_info; + dyad_ucx_request_t* req = NULL; + dyad_dtl_ucx_t* dtl_handle = self->private.ucx_dtl_handle; + FLUX_LOG_INFO (dtl_handle->h, "Poll UCP for incoming data\n"); + // TODO replace this loop with a resiliency response over RPC + do { + ucp_worker_progress (dtl_handle->ucx_worker); + msg = ucp_tag_probe_nb (dtl_handle->ucx_worker, + dtl_handle->comm_tag, + DYAD_UCX_TAG_MASK, + 1, // Remove the message from UCP tracking + // Requires calling ucp_tag_msg_recv_nb + // with the ucp_tag_message_h to retrieve message + &msg_info); + } while (msg == NULL); + // TODO: This version of the polling code is not supposed to spin-lock, + // unlike the code above. Currently, it does not work. Once it starts + // working, we can replace the code above with a version of this code. + // + // while (true) + // { + // // Probe the tag recv event at the top + // // of the worker's queue + // FLUX_LOG_INFO (dtl_handle->h, "Probe UCP worker with tag %lu\n", + // dtl_handle->comm_tag); msg = ucp_tag_probe_nb( + // dtl_handle->ucx_worker, + // dtl_handle->comm_tag, + // DYAD_UCX_TAG_MASK, + // 1, // Remove the message from UCP tracking + // // Requires calling ucp_tag_msg_recv_nb + // // with the ucp_tag_message_h to retrieve message + // &msg_info + // ); + // // If data has arrived from the producer plugin, + // // break the loop + // if (msg != NULL) + // { + // FLUX_LOG_INFO (dtl_handle->h, "Data has arrived, so end + // polling\n"); break; + // } + // // If data has not arrived, check if there are + // // any other events in the worker's queue. + // // If so, start the loop over to handle the next event + // else if (ucp_worker_progress(dtl_handle->ucx_worker)) + // { + // FLUX_LOG_INFO (dtl_handle->h, "Progressed UCP worker to check if + // any other UCP events are available\n"); continue; + // } + // // No other events are queued. So, we will wait on new + // // events to come in. By using 'ucp_worker_wait' for this, + // // we let the OS do other work in the meantime (no spin locking). + // FLUX_LOG_INFO (dtl_handle->h, "Launch pre-emptable wait until UCP + // worker gets new events\n"); status = + // ucp_worker_wait(dtl_handle->ucx_worker); + // // If the wait fails, log an error + // if (UCX_STATUS_FAIL(status)) + // { + // FLUX_LOG_ERR (dtl_handle->h, "Could not wait on the message from + // the producer plugin\n"); return DYAD_RC_UCXWAIT_FAIL; + // } + // } + // The metadata retrived from the probed tag recv event contains + // the size of the data to be sent. + // So, use that size to allocate a buffer + FLUX_LOG_INFO (dtl_handle->h, + "Got message with tag %lu and size %lu\n", + msg_info.sender_tag, + msg_info.length); + *buflen = msg_info.length; + *buf = malloc (*buflen); + // If allocation fails, log an error + if (*buf == NULL) { + FLUX_LOG_ERR (dtl_handle->h, "Could not allocate memory for file\n"); + return DYAD_RC_SYSFAIL; + } + FLUX_LOG_INFO (dtl_handle->h, "Receive data using async UCX operation\n"); +#if UCP_API_VERSION >= UCP_VERSION(1, 10) + // Define the settings for the recv operation + // + // The settings enabled are: + // * Define callback for the recv because it is async + // * Restrict memory buffers to host-only since we aren't directly + // dealing with GPU memory + ucp_request_param_t recv_params; + // TODO consider enabling UCP_OP_ATTR_FIELD_MEMH to speedup + // the recv operation if using RMA behind the scenes + recv_params.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_MEMORY_TYPE; + recv_params.cb.recv = dyad_recv_callback; + // Constraining to Host memory (as opposed to GPU memory) + // allows UCX to potentially perform some optimizations + recv_params.memory_type = UCS_MEMORY_TYPE_HOST; + // Perform the async recv operation using the probed tag recv event + req = ucp_tag_msg_recv_nbx (dtl_handle->ucx_worker, *buf, *buflen, msg, &recv_params); +#else + req = ucp_tag_msg_recv_nb (dtl_handle->ucx_worker, + *buf, + *buflen, + UCP_DATATYPE_CONTIG, + msg, + dyad_recv_callback); +#endif + // Wait on the recv operation to complete + FLUX_LOG_INFO (dtl_handle->h, "Wait for UCP recv operation to complete\n"); + status = dyad_ucx_request_wait (dtl_handle, req); + // If the recv operation failed, log an error, free the data buffer, + // and set the buffer pointer to NULL + if (UCX_STATUS_FAIL (status)) { + FLUX_LOG_ERR (dtl_handle->h, "UCX recv failed!\n"); + free (*buf); + *buf = NULL; + return DYAD_RC_UCXCOMM_FAIL; + } + FLUX_LOG_INFO (dtl_handle->h, "Data receive using UCX is successful\n"); + FLUX_LOG_INFO (dtl_handle->h, "Received %lu bytes from producer\n", *buflen); + return DYAD_RC_OK; +} + +dyad_rc_t dyad_dtl_ucx_close_connection (dyad_dtl_t* self) +{ + ucs_status_t status = UCS_OK; + ucs_status_ptr_t stat_ptr; + dyad_dtl_ucx_t* dtl_handle = self->private.ucx_dtl_handle; + if (dtl_handle->curr_comm_mode == DYAD_COMM_SEND) { + if (dtl_handle != NULL) { + if (dtl_handle->ep != NULL) { + // ucp_tag_send_sync_nbx is the prefered version of this send + // since UCX 1.9 However, some systems (e.g., Lassen) may have + // an older verison This conditional compilation will use + // ucp_tag_send_sync_nbx if using UCX 1.9+, and it will use the + // deprecated ucp_tag_send_sync_nb if using UCX < 1.9. + FLUX_LOG_INFO (dtl_handle->h, "Start async closing of UCP endpoint\n"); +#if UCP_API_VERSION >= UCP_VERSION(1, 10) + ucp_request_param_t close_params; + close_params.op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS; + close_params.flags = UCP_EP_CLOSE_FLAG_FORCE; + stat_ptr = ucp_ep_close_nbx (dtl_handle->ep, &close_params); +#else + // TODO change to FORCE if we decide to enable err handleing + // mode + stat_ptr = ucp_ep_close_nb (dtl_handle->ep, UCP_EP_CLOSE_MODE_FORCE); +#endif + FLUX_LOG_INFO (dtl_handle->h, "Wait for endpoint closing to finish\n"); + // Don't use dyad_ucx_request_wait here because ep_close behaves + // differently than other UCX calls + if (stat_ptr != NULL) { + // Endpoint close is in-progress. + // Wait until finished + if (UCS_PTR_IS_PTR (stat_ptr)) { + do { + ucp_worker_progress (dtl_handle->ucx_worker); + status = ucp_request_check_status (stat_ptr); + } while (status == UCS_INPROGRESS); + ucp_request_free (stat_ptr); + } + // An error occurred during endpoint closure + // However, the endpoint can no longer be used + // Get the status code for reporting + else { + status = UCS_PTR_STATUS (stat_ptr); + } + if (status != UCS_OK) { + FLUX_LOG_ERR (dtl_handle->h, + "Could not successfully close Endpoint " + "(status = %d)! However, endpoint was " + "released.\n", + status); + } + } + dtl_handle->ep = NULL; + } + // Sender doesn't have a consumer address at this time + // So, free the consumer address when closing the connection + if (dtl_handle->consumer_address != NULL) { + free (dtl_handle->consumer_address); + dtl_handle->consumer_address = NULL; + dtl_handle->addr_len = 0; + } + dtl_handle->comm_tag = 0; + } + FLUX_LOG_INFO (dtl_handle->h, "UCP endpoint close successful\n"); + return DYAD_RC_OK; + } else if (dtl_handle->curr_comm_mode == DYAD_COMM_RECV) { + // Since we're using tag send/recv, there's no need + // to explicitly close the connection. So, all we're + // doing here is setting the tag back to 0 (which cannot + // be valid for DYAD because DYAD won't send a file from + // one node to the same node). + dtl_handle->comm_tag = 0; + return DYAD_RC_OK; + } else { + FLUX_LOG_ERR (dtl_handle->h, + "Somehow, an invalid comm mode reached " + "'close_connection'\n"); + // TODO create new RC for this case + return DYAD_RC_BAD_COMM_MODE; + } +} + +dyad_rc_t dyad_dtl_ucx_finalize (dyad_dtl_t** self) +{ + dyad_dtl_ucx_t* dtl_handle = NULL; + dyad_rc_t rc = DYAD_RC_OK; + if (self == NULL || *self == NULL || (*self)->private.ucx_dtl_handle == NULL) { + return DYAD_RC_OK; + } + dtl_handle = (*self)->private.ucx_dtl_handle; + FLUX_LOG_INFO (dtl_handle->h, "Finalizing UCX DTL\n"); + if (dtl_handle->ep != NULL) { + dyad_dtl_ucx_close_connection (*self); + dtl_handle->ep = NULL; + } + // Release consumer address if not already released + if (dtl_handle->consumer_address != NULL) { + ucp_worker_release_address (dtl_handle->ucx_worker, dtl_handle->consumer_address); + dtl_handle->consumer_address = NULL; + } + // Release worker if not already released + if (dtl_handle->ucx_worker != NULL) { + ucp_worker_destroy (dtl_handle->ucx_worker); + dtl_handle->ucx_worker = NULL; + } + // Release context if not already released + if (dtl_handle->ucx_ctx != NULL) { + ucp_cleanup (dtl_handle->ucx_ctx); + dtl_handle->ucx_ctx = NULL; + } + // Flux handle should be released by the + // DYAD context, so it is not released here + dtl_handle->h = NULL; + // Free the handle and set to NULL to prevent double free + free (dtl_handle); + (*self)->private.ucx_dtl_handle = NULL; + return DYAD_RC_OK; +} diff --git a/src/dtl/ucx_dtl.h b/src/dtl/ucx_dtl.h new file mode 100644 index 00000000..9b2f87ba --- /dev/null +++ b/src/dtl/ucx_dtl.h @@ -0,0 +1,50 @@ +#ifndef DYAD_DTL_UCX_H +#define DYAD_DTL_UCX_H + +#include +#include + +#include "dyad_dtl_impl.h" + +struct dyad_dtl_ucx { + flux_t* h; + bool debug; + ucp_context_h ucx_ctx; + ucp_worker_h ucx_worker; + ucp_ep_h ep; + dyad_dtl_comm_mode_t curr_comm_mode; + ucp_address_t* consumer_address; + size_t addr_len; + ucp_tag_t comm_tag; +}; + +typedef struct dyad_dtl_ucx dyad_dtl_ucx_t; + +dyad_rc_t dyad_dtl_ucx_init (dyad_dtl_t* self, + dyad_dtl_mode_t mode, + flux_t* h, + bool debug); + +dyad_rc_t dyad_dtl_ucx_rpc_pack (dyad_dtl_t* restrict self, + const char* restrict upath, + uint32_t producer_rank, + json_t** restrict packed_obj); + +dyad_rc_t dyad_dtl_ucx_rpc_unpack (dyad_dtl_t* self, const flux_msg_t* msg, char** upath); + +dyad_rc_t dyad_dtl_ucx_rpc_respond (dyad_dtl_t* self, const flux_msg_t* orig_msg); + +dyad_rc_t dyad_dtl_ucx_rpc_recv_response (dyad_dtl_t* self, flux_future_t* f); + +dyad_rc_t dyad_dtl_ucx_establish_connection (dyad_dtl_t* self, + dyad_dtl_comm_mode_t comm_mode); + +dyad_rc_t dyad_dtl_ucx_send (dyad_dtl_t* self, void* buf, size_t buflen); + +dyad_rc_t dyad_dtl_ucx_recv (dyad_dtl_t* self, void** buf, size_t* buflen); + +dyad_rc_t dyad_dtl_ucx_close_connection (dyad_dtl_t* self); + +dyad_rc_t dyad_dtl_ucx_finalize (dyad_dtl_t** self); + +#endif /* DYAD_DTL_UCX_H */ \ No newline at end of file diff --git a/src/modules/Makefile-urpc b/src/modules/Makefile-urpc deleted file mode 100644 index e55b367c..00000000 --- a/src/modules/Makefile-urpc +++ /dev/null @@ -1,40 +0,0 @@ -CONFIG = ../urpc.cfg -include ${CONFIG} - -ifeq ($(URPC_DEBUG),1) - URPC_OPTIONS += -DURPC_FULL_DEBUG=1 - URPC_OPTIONS += -DURPC_LOGGING_ON=1 - $(info URPC_DEBUG is enabled) -endif - -all: urpc.so - +$(MAKE) -C test - -LIBB64_DIR = /p/lustre2/yeom2/FLUX/libb64 -_CFLAGS = $(CFLAGS) $(C_VER) $(URPC_OPTIONS) -I../common -I$(FLUX_CORE_INCLUDES) -I$(LIBB64_DIR)/include -_LDFLAGS += $(LDFLAGS) -L$(FLUX_CORE_LIBPATH) -Wl,-rpath=$(FLUX_CORE_LIBPATH),--no-undefined -shared $(FLUX_CORE_LIBS) -ldl -ljansson -L$(LIBB64_DIR)/src -lb64 -# --disable-static - -urpc.so: urpc.o ../common/utils.o read_all.o - $(CC) $(_CFLAGS) $^ -o $@ $(_LDFLAGS) - #$(CC) $(_CFLAGS) -L$(FLUX_CORE_LIBPATH) -Wl,-rpath=$(FLUX_CORE_LIBPATH),--no-undefined -shared $^ -o $@ $(FLUX_CORE_LIBS) -ldl - -urpc.o: urpc.c urpc_ctx.h - $(CC) $(_CFLAGS) $< -c -o $@ - -read_all.o: read_all.c - $(CC) $(_CFLAGS) $^ -c -o $@ - -start: urpc.so - flux module load ./urpc.so $(shell readlink -f ../wrapper/test)/prod - - -.PHONY: clean install - -install: urpc.so - install -d ${PREFIX}/lib - install -m 640 urpc.so ${PREFIX}/lib - -clean: - @rm -f *.o *.so - +$(MAKE) clean -C test diff --git a/src/modules/Makefile.am b/src/modules/Makefile.am index a57f214e..56c0e3e8 100644 --- a/src/modules/Makefile.am +++ b/src/modules/Makefile.am @@ -1,21 +1,40 @@ lib_LTLIBRARIES = dyad.la -dyad_la_SOURCES = dyad.c read_all.c dyad_ctx.h read_all.h +dyad_la_SOURCES = \ + dyad.c dyad_la_LDFLAGS = \ $(AM_LDFLAGS) \ -module \ -avoid-version \ -no-undefined \ - -export-symbols-regex '^mod_(name|main)$$' -dyad_la_LIBADD = $(top_builddir)/src/utils/libutils.la $(FLUX_CORE_LIBS) -dyad_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_builddir)/src/utils $(FLUX_CORE_CFLAGS) + --disable-static \ + -shared \ + -export-dynamic \ + -Wl,-rpath,'$(UCX_LIBDIR)' +dyad_la_LIBADD = \ + $(top_builddir)/src/dtl/libdyad_dtl.la \ + $(UCX_LIBS) \ + $(JANSSON_LIBS) \ + $(FLUX_CORE_LIBS) +dyad_la_CFLAGS = \ + $(AM_CFLAGS) \ + -I$(top_srcdir)/src/utils \ + -I$(top_srcdir)/src/utils/base64 \ + -I$(top_srcdir)/src/dtl \ + $(UCX_CFLAGS) \ + $(JANSSON_CFLAGS) \ + $(FLUX_CORE_CFLAGS) \ + -DBUILDING_DYAD \ + -fvisibility=hidden +dyad_la_CPPFLAGS = if PERFFLOW dyad_la_LIBADD += $(PERFFLOW_LIBS) -dyad_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS) $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1 +dyad_la_CFLAGS += $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1 +dyad_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS) endif if URPC lib_LTLIBRARIES += urpc.la -urpc_la_SOURCES = read_all.c urpc.c urpc_ctx.h +urpc_la_SOURCES = urpc.c urpc_ctx.h # TODO replace current LDFLAGS rule with this one once libb64 is required # urpc_la_LDFLAGS = -module -avoid-version -no-undefined -L$(LIBB64_DIR)/src urpc_la_LDFLAGS = -module -avoid-version -no-undefined @@ -24,12 +43,14 @@ urpc_la_LIBADD = $(top_builddir)/src/utils/libutils.la $(FLUX_CORE_LIBS) $(JANSS # urpc_la_LIBS = -lb64 # TODO replace current CPPFLAGS rule with this one once libb64 is required # urpc_la_CPPFLAGS = $(FLUX_CORE_CFLAGS) $(JANSSON_CFLAGS) -I$(LIBB64_DIR)/include -I$(top_builddir)/src/common -urpc_la_CPPFLAGS = $(FLUX_CORE_CFLAGS) $(JANSSON_CFLAGS) -I$(top_builddir)/src/utils +urpc_la_CFLAGS = $(FLUX_CORE_CFLAGS) $(JANSSON_CFLAGS) -I$(top_builddir)/src/utils +urpc_la_CPPFLAGS = if PERFFLOW urpc_la_LIBADD += $(PERFFLOW_LIBS) -urpc_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS) $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1 +urpc_la_CFLAGS += $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1 +urpc_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS) endif endif install-exec-hook: - @(cd $(DESTDIR)$(libdir) && $(RM) dyad.la dyad.a) + @(cd $(DESTDIR)$(libdir) && $(RM) dyad.la) diff --git a/src/modules/dyad.c b/src/modules/dyad.c index 036f077b..af7ac2bf 100644 --- a/src/modules/dyad.c +++ b/src/modules/dyad.c @@ -17,6 +17,7 @@ #include #else #include +#include #include #include #include @@ -25,12 +26,12 @@ #endif // defined(__cplusplus) #include -#include #include #include #include -#include "dyad_ctx.h" +#include "dyad_dtl_impl.h" +#include "dyad_rc.h" #include "read_all.h" #include "utils.h" @@ -39,17 +40,17 @@ - (Tstart).tv_nsec) \ / 1000000000L) -#if !defined(DYAD_LOGGING_ON) || (DYAD_LOGGING_ON == 0) -#define FLUX_LOG_INFO(...) \ - do { \ - } while (0) -#define FLUX_LOG_ERR(...) \ - do { \ - } while (0) -#else -#define FLUX_LOG_INFO(h, ...) flux_log (h, LOG_INFO, __VA_ARGS__) -#define FLUX_LOG_ERR(h, ...) flux_log_error (h, __VA_ARGS__) -#endif +struct dyad_mod_ctx { + flux_t *h; + bool debug; + flux_msg_handler_t **handlers; + const char *dyad_path; + dyad_dtl_t *dtl_handle; +}; + +const struct dyad_mod_ctx dyad_mod_ctx_default = {NULL, false, NULL, NULL, NULL}; + +typedef struct dyad_mod_ctx dyad_mod_ctx_t; static void dyad_mod_fini (void) __attribute__ ((destructor)); @@ -65,6 +66,10 @@ static void freectx (void *arg) { dyad_mod_ctx_t *ctx = (dyad_mod_ctx_t *)arg; flux_msg_handler_delvec (ctx->handlers); + if (ctx->dtl_handle != NULL) { + dyad_dtl_finalize (&(ctx->dtl_handle)); + ctx->dtl_handle = NULL; + } free (ctx); } @@ -74,50 +79,38 @@ static dyad_mod_ctx_t *getctx (flux_t *h) if (!ctx) { ctx = (dyad_mod_ctx_t *)malloc (sizeof (*ctx)); + if (ctx == NULL) { + FLUX_LOG_ERR (h, "DYAD_MOD: could not allocate memory for context"); + goto getctx_error; + } ctx->h = h; ctx->debug = false; ctx->handlers = NULL; ctx->dyad_path = NULL; + ctx->dtl_handle = NULL; if (flux_aux_set (h, "dyad", ctx, freectx) < 0) { FLUX_LOG_ERR (h, "DYAD_MOD: flux_aux_set() failed!\n"); - goto error; + goto getctx_error; } } - goto done; + goto getctx_done; -error:; +getctx_error:; return NULL; -done: +getctx_done: return ctx; } -#if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) void dyad_respond ( - flux_t *h, - const flux_msg_t *msg, - const void *inbuf, - size_t inlen) -{ - if (flux_respond_raw (h, msg, inbuf, inlen) < 0) { - FLUX_LOG_ERR (h, "DYAD_MOD: %s: flux_respond", __FUNCTION__); - } else { - FLUX_LOG_INFO (h, "DYAD_MOD: dyad_fetch_request_cb() served\n"); - } -} -#endif // DYAD_PERFFLOW - /* request callback called when dyad.fetch request is invoked */ #if DYAD_PERFFLOW __attribute__ ((annotate ("@critical_path()"))) #endif static void -dyad_fetch_request_cb (flux_t *h, - flux_msg_handler_t *w, - const flux_msg_t *msg, - void *arg) +dyad_fetch_request_cb (flux_t *h, flux_msg_handler_t *w, const flux_msg_t *msg, void *arg) { + FLUX_LOG_INFO (h, "Launched callback for dyad.fetch\n"); dyad_mod_ctx_t *ctx = getctx (h); ssize_t inlen = 0; void *inbuf = NULL; @@ -126,134 +119,209 @@ dyad_fetch_request_cb (flux_t *h, char *upath = NULL; char fullpath[PATH_MAX + 1] = {'\0'}; int saved_errno = errno; - errno = 0; + dyad_rc_t rc = 0; + + if (!flux_msg_is_streaming (msg)) { + errno = EPROTO; + goto fetch_error; + } if (flux_msg_get_userid (msg, &userid) < 0) - goto error; + goto fetch_error; - if (flux_request_unpack (msg, NULL, "{s:s}", "upath", &upath) < 0) - goto error; + FLUX_LOG_INFO (h, "DYAD_MOD: unpacking RPC message"); + + rc = ctx->dtl_handle->rpc_unpack (ctx->dtl_handle, msg, &upath); + + if (DYAD_IS_ERROR (rc)) { + FLUX_LOG_ERR (ctx->h, "Could not unpack message from client\n"); + errno = EPROTO; + goto fetch_error; + } FLUX_LOG_INFO (h, "DYAD_MOD: requested user_path: %s", upath); + FLUX_LOG_INFO (h, "DYAD_MOD: sending initial response to consumer"); + + rc = ctx->dtl_handle->rpc_respond (ctx->dtl_handle, msg); + if (DYAD_IS_ERROR (rc)) { + FLUX_LOG_ERR (ctx->h, "Could not send primary RPC response to client\n"); + goto fetch_error; + } strncpy (fullpath, ctx->dyad_path, PATH_MAX - 1); concat_str (fullpath, upath, "/", PATH_MAX); #if DYAD_SPIN_WAIT if (!get_stat (fullpath, 1000U, 1000L)) { - FLUX_LOG_ERR (h, "DYAD_MOD: Failed to access info on \"%s\".\n", - fullpath); + FLUX_LOG_ERR (h, "DYAD_MOD: Failed to access info on \"%s\".\n", fullpath); // goto error; } #endif // DYAD_SPIN_WAIT + FLUX_LOG_INFO (h, "Reading file %s for transfer", fullpath); fd = open (fullpath, O_RDONLY); if (fd < 0) { FLUX_LOG_ERR (h, "DYAD_MOD: Failed to open file \"%s\".\n", fullpath); - goto error; + goto fetch_error; } if ((inlen = read_all (fd, &inbuf)) < 0) { FLUX_LOG_ERR (h, "DYAD_MOD: Failed to load file \"%s\".\n", fullpath); - goto error; + goto fetch_error; } close (fd); + FLUX_LOG_INFO (h, "Is inbuf NULL? -> %i\n", (int)(inbuf == NULL)); + + FLUX_LOG_INFO (h, "Establish DTL connection with consumer"); + rc = ctx->dtl_handle->establish_connection (ctx->dtl_handle, DYAD_COMM_SEND); + if (DYAD_IS_ERROR (rc)) { + FLUX_LOG_ERR (ctx->h, "Could not establish DTL connection with client\n"); + errno = ECONNREFUSED; + goto fetch_error; + } + FLUX_LOG_INFO (h, "Send file to consumer with DTL"); + rc = ctx->dtl_handle->send (ctx->dtl_handle, inbuf, inlen); + FLUX_LOG_INFO (h, "Close DTL connection with consumer"); + ctx->dtl_handle->close_connection (ctx->dtl_handle); + free (inbuf); + if (DYAD_IS_ERROR (rc)) { + FLUX_LOG_ERR (ctx->h, "Could not send data to client via DTL\n"); + errno = ECOMM; + goto fetch_error; + } - goto done; - -error: - if (flux_respond_error (h, msg, errno, NULL) < 0) { - FLUX_LOG_ERR (h, "DYAD_MOD: %s: flux_respond_error", __FUNCTION__); + FLUX_LOG_INFO (h, "Close RPC message stream with an ENODATA (%d) message", ENODATA); + if (flux_respond_error (h, msg, ENODATA, NULL) < 0) { + FLUX_LOG_ERR (h, + "DYAD_MOD: %s: flux_respond_error with ENODATA failed\n", + __FUNCTION__); } + errno = saved_errno; + FLUX_LOG_INFO (h, "Finished dyad.fetch module invocation\n"); return; -done: -#if DYAD_PERFFLOW - dyad_respond (h, msg, inbuf, inlen); -#else - // if (flux_respond_raw (h, msg, errno, inbuf, inlen) < 0) - if (flux_respond_raw (h, msg, inbuf, inlen) < 0) { - FLUX_LOG_ERR (h, "DYAD_MOD: %s: flux_respond", __FUNCTION__); - } else { - FLUX_LOG_INFO (h, "DYAD_MOD: dyad_fetch_request_cb() served %s\n", - fullpath); +fetch_error: + FLUX_LOG_ERR (h, "Close RPC message stream with an error (errno = %d)\n", errno); + if (flux_respond_error (h, msg, errno, NULL) < 0) { + FLUX_LOG_ERR (h, "DYAD_MOD: %s: flux_respond_error", __FUNCTION__); } - // TODO: check if flux_respond_raw deallocates inbuf. - // If not, deallocate it here -#endif // DYAD_PERFFLOW errno = saved_errno; return; } -static int dyad_open (flux_t *h) +static dyad_rc_t dyad_open (flux_t *h, dyad_dtl_mode_t dtl_mode, bool debug) { dyad_mod_ctx_t *ctx = getctx (h); - int rc = -1; + dyad_rc_t rc = 0; char *e = NULL; - if ((e = getenv ("DYAD_MOD_DEBUG")) && atoi (e)) - ctx->debug = true; - rc = 0; + ctx->debug = debug; + rc = dyad_dtl_init (&(ctx->dtl_handle), dtl_mode, h, ctx->debug); return rc; } -static const struct flux_msg_handler_spec htab[] = {{FLUX_MSGTYPE_REQUEST, - "dyad.fetch", - dyad_fetch_request_cb, 0}, - FLUX_MSGHANDLER_TABLE_END}; +static const struct flux_msg_handler_spec htab[] = + {{FLUX_MSGTYPE_REQUEST, DYAD_DTL_RPC_NAME, dyad_fetch_request_cb, 0}, + FLUX_MSGHANDLER_TABLE_END}; -int mod_main (flux_t *h, int argc, char **argv) +void usage () +{ + fprintf (stderr, + "Usage: flux exec -r all flux module load dyad.so " + "[DTL_MODE] [--debug | -d]\n\n"); + fprintf (stderr, "Required Arguments:\n"); + fprintf (stderr, "===================\n"); + fprintf (stderr, + " * PRODUCER_PATH: the producer-managed path that the module should " + "track\n\n"); + fprintf (stderr, "Optional Arguments:\n"); + fprintf (stderr, "===================\n"); + fprintf (stderr, + " * DTL_MODE: a valid data transport layer (DTL) mode. Can be one of the " + "following values\n"); + fprintf (stderr, " * UCX (default): use UCX to send data to consumer\n"); + fprintf (stderr, + " * FLUX_RPC: use Flux's RPC response mechanism to send data to " + "consumer\n"); + fprintf (stderr, " * --debug | -d: if provided, add debugging log messages\n"); +} + +DYAD_DLL_EXPORTED int mod_main (flux_t *h, int argc, char **argv) { const mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); dyad_mod_ctx_t *ctx = NULL; + size_t flag_len = 0; + dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; + bool debug = false; if (!h) { fprintf (stderr, "Failed to get flux handle\n"); - goto done; + goto mod_done; } ctx = getctx (h); - if (argc != 1) { + if (argc < 1) { FLUX_LOG_ERR (ctx->h, - "DYAD_MOD: Missing argument. " - "Requires a local dyad path specified.\n"); - fprintf (stderr, - "Missing argument. Requires a local dyad path specified.\n"); - goto error; + "DYAD_MOD: Missing argument(s). " + "Requires a local dyad path.\n"); + usage (); + goto mod_error; } (ctx->dyad_path) = argv[0]; mkdir_as_needed (ctx->dyad_path, m); - if (dyad_open (h) < 0) { + if (argc >= 2) { + FLUX_LOG_INFO (h, "DTL Mode (from cmd line) is %s\n", argv[1]); + flag_len = strlen (argv[1]); + if (strncmp (argv[1], "FLUX_RPC", flag_len) == 0) { + dtl_mode = DYAD_DTL_FLUX_RPC; + } else if (strncmp (argv[1], "UCX", flag_len) == 0) { + dtl_mode = DYAD_DTL_UCX; + } else { + FLUX_LOG_ERR (ctx->h, "Invalid DTL mode provided\n"); + usage (); + goto mod_error; + } + } + + if (argc >= 3) { + flag_len = strlen (argv[2]); + if (strncmp (argv[2], "--debug", flag_len) == 0 + || strncmp (argv[2], "-d", flag_len) == 0) { + debug = true; + } else { + debug = false; + } + } + + if (DYAD_IS_ERROR (dyad_open (h, dtl_mode, debug))) { FLUX_LOG_ERR (ctx->h, "dyad_open failed"); - goto error; + goto mod_error; } - fprintf (stderr, "dyad module begins using \"%s\"\n", argv[0]); FLUX_LOG_INFO (ctx->h, "dyad module begins using \"%s\"\n", argv[0]); if (flux_msg_handler_addvec (ctx->h, htab, (void *)h, &ctx->handlers) < 0) { - FLUX_LOG_ERR (ctx->h, "flux_msg_handler_addvec: %s\n", - strerror (errno)); - goto error; + FLUX_LOG_ERR (ctx->h, "flux_msg_handler_addvec: %s\n", strerror (errno)); + goto mod_error; } if (flux_reactor_run (flux_get_reactor (ctx->h), 0) < 0) { FLUX_LOG_ERR (ctx->h, "flux_reactor_run: %s", strerror (errno)); - goto error; + goto mod_error; } - goto done; + goto mod_done; -error:; +mod_error:; return EXIT_FAILURE; -done:; +mod_done:; return EXIT_SUCCESS; } -MOD_NAME ("dyad"); +DYAD_DLL_EXPORTED MOD_NAME ("dyad"); /* * vi:tabstop=4 shiftwidth=4 expandtab diff --git a/src/modules/dyad_ctx.h b/src/modules/dyad_ctx.h deleted file mode 100644 index 23a80eae..00000000 --- a/src/modules/dyad_ctx.h +++ /dev/null @@ -1,26 +0,0 @@ -/************************************************************\ - * Copyright 2021 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#ifndef DYAD_MODULES_DYAD_CTX_H -#define DYAD_MODULES_DYAD_CTX_H - -#include -#include - -struct dyad_mod_ctx { - flux_t *h; - bool debug; - flux_msg_handler_t **handlers; - const char *dyad_path; -} dyad_mod_ctx_default = {NULL, false, NULL, NULL}; - -typedef struct dyad_mod_ctx dyad_mod_ctx_t; - -#endif // DYAD_MODULES_DYAD_CTX_H diff --git a/src/modules/test/compute.c b/src/modules/test/compute.c index 4f4e4217..5ba4efc2 100644 --- a/src/modules/test/compute.c +++ b/src/modules/test/compute.c @@ -51,14 +51,12 @@ static int publish_via_flux (FILE *df, flux_t *h, const char *user_path) goto done; } if (flux_kvs_txn_pack (txn, 0, topic, "i", owner_rank) < 0) { - fprintf (df, "flux_kvs_txn_pack(\"%s\",\"i\",%u) failed.\n", topic, - owner_rank); + fprintf (df, "flux_kvs_txn_pack(\"%s\",\"i\",%u) failed.\n", topic, owner_rank); goto done; } if (!(f = flux_kvs_commit (h, getenv ("DYAD_KVS_NAMESPACE"), 0, txn))) { - fprintf (df, "flux_kvs_commit(owner rank of %s = %u)\n", user_path, - owner_rank); + fprintf (df, "flux_kvs_commit(owner rank of %s = %u)\n", user_path, owner_rank); goto done; } @@ -109,8 +107,7 @@ int main (int argc, char **argv) strncpy (topic, file_name, topic_len); fprintf (df, "Consumer kvs search for %s\n", file_name); - f1 = flux_kvs_lookup (h, getenv ("DYAD_KVS_NAMESPACE"), FLUX_KVS_WAITCREATE, - topic); + f1 = flux_kvs_lookup (h, getenv ("DYAD_KVS_NAMESPACE"), FLUX_KVS_WAITCREATE, topic); if (f1 == NULL) { flux_log_error (h, "flux_kvs_lookup(%s) failed.\n", topic); goto done; @@ -127,7 +124,12 @@ int main (int argc, char **argv) fprintf (df, "Owner rank: %u\n", owner_rank); // Send the request to fetch a file - if (!(f2 = flux_rpc_pack (h, "dyad.fetch", owner_rank, 0, "{s:s}", "upath", + if (!(f2 = flux_rpc_pack (h, + "dyad.fetch", + owner_rank, + 0, + "{s:s}", + "upath", file_name))) { flux_log_error (h, "flux_rpc_pack({dyad.fetch %s})", file_name); goto done; @@ -154,8 +156,7 @@ int main (int argc, char **argv) // Create the directory as needed const char *odir = dirname (file_path_copy); const mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); - if ((strncmp (odir, ".", strlen (".")) != 0) - && (mkdir_as_needed (odir, m) < 0)) { + if ((strncmp (odir, ".", strlen (".")) != 0) && (mkdir_as_needed (odir, m) < 0)) { fprintf (stderr, "Failed to create directory \"%s\".\n", odir); goto done; } diff --git a/src/modules/urpc.c b/src/modules/urpc.c index f51d5e81..b79bd3c2 100644 --- a/src/modules/urpc.c +++ b/src/modules/urpc.c @@ -106,8 +106,7 @@ urpc_exec_cmd (flux_t *h, const char *cmd, void **inbuf, ssize_t *inlen) fd = fileno (fp); if ((*inlen = read_all (fd, inbuf)) < 0) { - FLUX_LOG_ERR (h, "URPC_MOD: Failed to catch result from \"%s\".\n", - cmd); + FLUX_LOG_ERR (h, "URPC_MOD: Failed to catch result from \"%s\".\n", cmd); goto error; } @@ -140,10 +139,7 @@ void urpc_respond (flux_t *h, const flux_msg_t *msg, const void *inbuf, size_t i __attribute__ ((annotate ("@critical_path()"))) #endif static void -urpc_exec_request_cb (flux_t *h, - flux_msg_handler_t *w, - const flux_msg_t *msg, - void *arg) +urpc_exec_request_cb (flux_t *h, flux_msg_handler_t *w, const flux_msg_t *msg, void *arg) { // urpc_server_ctx_t *ctx = getctx (h); ssize_t inlen = 0; @@ -188,10 +184,7 @@ urpc_exec_request_cb (flux_t *h, __attribute__ ((annotate ("@critical_path()"))) #endif static void -urpc_execj_request_cb (flux_t *h, - flux_msg_handler_t *w, - const flux_msg_t *msg, - void *arg) +urpc_execj_request_cb (flux_t *h, flux_msg_handler_t *w, const flux_msg_t *msg, void *arg) { // urpc_server_ctx_t *ctx = getctx (h); ssize_t inlen = 0; @@ -223,14 +216,22 @@ urpc_execj_request_cb (flux_t *h, } if (!(jcmd = json_loads (cmd_json, 0, &error))) { - FLUX_LOG_ERR (h, "URPC_MOD: json error on line %d: %s\n", error.line, - error.text); + FLUX_LOG_ERR (h, "URPC_MOD: json error on line %d: %s\n", error.line, error.text); goto error; } // TODO: generalize the format string. - rc = json_unpack (jcmd, "{s:s, s:[s, {s:s, s:s}, i]}", "cmd", &exec, "args", - &arg1, "file", &filename, "content", &content, &n); + rc = json_unpack (jcmd, + "{s:s, s:[s, {s:s, s:s}, i]}", + "cmd", + &exec, + "args", + &arg1, + "file", + &filename, + "content", + &content, + &n); if (rc) { FLUX_LOG_ERR (h, "URPC_MOD: could not unpack '%s'.\n", cmd_json); @@ -314,8 +315,7 @@ int mod_main (flux_t *h, int argc, char **argv) FLUX_LOG_INFO (ctx->h, "urpc module begins using \"%s\"\n", argv[0]); if (flux_msg_handler_addvec (ctx->h, htab, (void *)h, &ctx->handlers) < 0) { - FLUX_LOG_ERR (ctx->h, "flux_msg_handler_addvec: %s\n", - strerror (errno)); + FLUX_LOG_ERR (ctx->h, "flux_msg_handler_addvec: %s\n", strerror (errno)); goto error; } diff --git a/src/stream/Makefile.am b/src/stream/Makefile.am index 5d958519..a34ffab0 100644 --- a/src/stream/Makefile.am +++ b/src/stream/Makefile.am @@ -1,12 +1,23 @@ lib_LTLIBRARIES = libdyad_fstream.la libdyad_fstream_la_SOURCES = dyad_stream_core.cpp -libdyad_fstream_la_LDFLAGS = $(AM_LDFLAGS) -avoid-version -no-undefined +libdyad_fstream_la_LDFLAGS = \ + -Wl,-rpath,'$(UCX_LIBDIR)' \ + $(AM_LDFLAGS) \ + -avoid-version \ + -no-undefined libdyad_fstream_la_LIBADD = $(top_builddir)/src/core/libdyad_core.la -libdyad_fstream_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_builddir)/src/utils -I$(top_builddir)/src/core $(FLUX_CORE_CFLAGS) +libdyad_fstream_la_CXXFLAGS = \ + $(AM_CFLAGS) \ + -I$(top_srcdir)/src/utils \ + -I$(top_srcdir)/src/core \ + -I$(top_srcdir)/src/dtl \ + $(FLUX_CORE_CFLAGS) +libdyad_fstream_la_CPPFLAGS = if PERFFLOW libdyad_fstream_la_LIBADD += $(PERFFLOW_LIBS) -libdyad_fstream_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS) $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1 +libdyad_fstream_la_CXXFLAGS += $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1 +libdyad_fstream_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS) endif include_HEADERS = dyad_params.hpp dyad_stream_core.hpp dyad_stream_api.hpp diff --git a/src/stream/dyad_params.hpp b/src/stream/dyad_params.hpp index 062aa3cb..8e1cba00 100644 --- a/src/stream/dyad_params.hpp +++ b/src/stream/dyad_params.hpp @@ -24,6 +24,9 @@ struct dyad_params { unsigned int m_key_depth; /// The number of bins used in key hashing unsigned int m_key_bins; + /// The DTL to use to move data from producer to consumer + /// Valid values can be found in dyad_dtl_defs.h from core + int m_dtl_mode; /// The KVS namespace of the sharing context std::string m_kvs_namespace; @@ -38,6 +41,7 @@ struct dyad_params { m_shared_storage (false), m_key_depth (2u), m_key_bins (256u), + m_dtl_mode (0), m_kvs_namespace (""), m_cons_managed_path (""), m_prod_managed_path ("") diff --git a/src/stream/dyad_stream_api.hpp b/src/stream/dyad_stream_api.hpp index 09a6880b..9cec061b 100644 --- a/src/stream/dyad_stream_api.hpp +++ b/src/stream/dyad_stream_api.hpp @@ -12,7 +12,7 @@ #define DYAD_STREAM_DYAD_STREAM_API_HPP #include // fsync -#include // realpath +#include // realpath #include #include #include @@ -43,8 +43,7 @@ template ().make_preferred ().filename ())> -using dyad_if_fs_path = - std::enable_if_t, _Result>; +using dyad_if_fs_path = std::enable_if_t, _Result>; #endif // c++17 filesystem // https://stackoverflow.com/questions/676787/how-to-do-fsync-on-an-ofstream @@ -149,11 +148,9 @@ class basic_ifstream_dyad using ifstream_dyad = basic_ifstream_dyad; using wifstream_dyad = basic_ifstream_dyad; -#if __cplusplus \ - < 201103L //---------------------------------------------------- +#if __cplusplus < 201103L //---------------------------------------------------- template -basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad ( - const dyad_stream_core& core) +basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad (const dyad_stream_core& core) : m_core (core) { m_stream = new basic_ifstream (); @@ -167,9 +164,8 @@ basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad () } template -basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad ( - const char* filename, - std::ios_base::openmode mode) +basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad (const char* filename, + std::ios_base::openmode mode) { m_core.init (); m_core.open_sync (filename); @@ -186,8 +182,7 @@ bool basic_ifstream_dyad<_CharT, _Traits>::is_open () } #else //----------------------------------------------------------------------- template -basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad ( - const dyad_stream_core& core) +basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad (const dyad_stream_core& core) : m_core (core) { m_stream = std::unique_ptr (new basic_ifstream ()); @@ -201,36 +196,31 @@ basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad () } template -basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad ( - const char* filename, - std::ios_base::openmode mode) +basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad (const char* filename, + std::ios_base::openmode mode) { m_core.init (); m_core.open_sync (filename); - m_stream = - std::unique_ptr (new basic_ifstream (filename, mode)); + m_stream = std::unique_ptr (new basic_ifstream (filename, mode)); if ((m_stream != nullptr) && (*m_stream)) { m_filename = std::string{filename}; } } template -basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad ( - const string& filename, - std::ios_base::openmode mode) +basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad (const string& filename, + std::ios_base::openmode mode) { m_core.init (); m_core.open_sync (filename.c_str ()); - m_stream = - std::unique_ptr (new basic_ifstream (filename, mode)); + m_stream = std::unique_ptr (new basic_ifstream (filename, mode)); if ((m_stream != nullptr) && (*m_stream)) { m_filename = filename; } } template -basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad ( - basic_ifstream_dyad&& rhs) +basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad (basic_ifstream_dyad&& rhs) : m_stream (std::move (rhs.m_stream)), m_core (std::move (rhs.m_core)) { } @@ -238,20 +228,19 @@ basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad ( #if (__cplusplus >= 201703L) && __has_include() template template -basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad ( - const _Path& filepath, - std::ios_base::openmode mode) +basic_ifstream_dyad<_CharT, _Traits>::basic_ifstream_dyad (const _Path& filepath, + std::ios_base::openmode mode) { m_core.init (); m_core.open_sync (filepath.c_str ()); - m_stream = std::unique_ptr ( - new basic_ifstream (filepath.c_str (), mode)); + m_stream = + std::unique_ptr (new basic_ifstream (filepath.c_str (), mode)); } #endif // c++17 filesystem template -basic_ifstream_dyad<_CharT, _Traits>& basic_ifstream_dyad<_CharT, _Traits>:: -operator= (basic_ifstream_dyad&& rhs) +basic_ifstream_dyad<_CharT, _Traits>& basic_ifstream_dyad<_CharT, _Traits>::operator= ( + basic_ifstream_dyad&& rhs) { m_stream = std::move (rhs.m_stream); m_core = std::move (rhs.m_core); @@ -330,8 +319,8 @@ std::filebuf* basic_ifstream_dyad<_CharT, _Traits>::rdbuf () const } template -std::basic_ifstream<_CharT, _Traits>& basic_ifstream_dyad<_CharT, _Traits>:: - get_stream () +std::basic_ifstream<_CharT, _Traits>& basic_ifstream_dyad<_CharT, + _Traits>::get_stream () { if (m_stream == nullptr) { // TODO: throw @@ -412,11 +401,9 @@ class basic_ofstream_dyad using ofstream_dyad = basic_ofstream_dyad; using wofstream_dyad = basic_ofstream_dyad; -#if __cplusplus \ - < 201103L //---------------------------------------------------- +#if __cplusplus < 201103L //---------------------------------------------------- template -basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad ( - const dyad_stream_core& core) +basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const dyad_stream_core& core) : m_core (core) { m_stream = new basic_ofstream (); @@ -430,9 +417,8 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad () } template -basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad ( - const char* filename, - std::ios_base::openmode mode) +basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const char* filename, + std::ios_base::openmode mode) { m_core.init (); m_stream = new basic_ofstream (filename, mode); @@ -448,8 +434,7 @@ bool basic_ofstream_dyad<_CharT, _Traits>::is_open () } #else //----------------------------------------------------------------------- template -basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad ( - const dyad_stream_core& core) +basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const dyad_stream_core& core) : m_core (core) { m_stream = std::unique_ptr (new basic_ofstream ()); @@ -463,34 +448,29 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad () } template -basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad ( - const char* filename, - std::ios_base::openmode mode) +basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const char* filename, + std::ios_base::openmode mode) { m_core.init (); - m_stream = - std::unique_ptr (new basic_ofstream (filename, mode)); + m_stream = std::unique_ptr (new basic_ofstream (filename, mode)); if ((m_stream != nullptr) && (*m_stream)) { m_filename = std::string{filename}; } } template -basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad ( - const string& filename, - std::ios_base::openmode mode) +basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const string& filename, + std::ios_base::openmode mode) { m_core.init (); - m_stream = - std::unique_ptr (new basic_ofstream (filename, mode)); + m_stream = std::unique_ptr (new basic_ofstream (filename, mode)); if ((m_stream != nullptr) && (*m_stream)) { m_filename = filename; } } template -basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad ( - basic_ofstream_dyad&& rhs) +basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (basic_ofstream_dyad&& rhs) : m_stream (std::move (rhs.m_stream)), m_core (std::move (rhs.m_core)) { } @@ -498,19 +478,18 @@ basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad ( #if (__cplusplus >= 201703L) && __has_include() template template -basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad ( - const _Path& filepath, - std::ios_base::openmode mode) +basic_ofstream_dyad<_CharT, _Traits>::basic_ofstream_dyad (const _Path& filepath, + std::ios_base::openmode mode) { m_core.init (); - m_stream = std::unique_ptr ( - new basic_ofstream (filepath.c_str (), mode)); + m_stream = + std::unique_ptr (new basic_ofstream (filepath.c_str (), mode)); } #endif // c++17 filesystem template -basic_ofstream_dyad<_CharT, _Traits>& basic_ofstream_dyad<_CharT, _Traits>:: -operator= (basic_ofstream_dyad&& rhs) +basic_ofstream_dyad<_CharT, _Traits>& basic_ofstream_dyad<_CharT, _Traits>::operator= ( + basic_ofstream_dyad&& rhs) { m_stream = std::move (rhs.m_stream); m_core = std::move (rhs.m_core); @@ -601,8 +580,8 @@ std::filebuf* basic_ofstream_dyad<_CharT, _Traits>::rdbuf () const } template -std::basic_ofstream<_CharT, _Traits>& basic_ofstream_dyad<_CharT, _Traits>:: - get_stream () +std::basic_ofstream<_CharT, _Traits>& basic_ofstream_dyad<_CharT, + _Traits>::get_stream () { if (m_stream == nullptr) { // TODO: throw @@ -687,11 +666,9 @@ class basic_fstream_dyad using fstream_dyad = basic_fstream_dyad; using wfstream_dyad = basic_fstream_dyad; -#if __cplusplus \ - < 201103L //---------------------------------------------------- +#if __cplusplus < 201103L //---------------------------------------------------- template -basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad ( - const dyad_stream_core& core) +basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (const dyad_stream_core& core) : m_core (core) { m_stream = new basic_fstream (); @@ -705,9 +682,8 @@ basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad () } template -basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad ( - const char* filename, - std::ios_base::openmode mode) +basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (const char* filename, + std::ios_base::openmode mode) { m_core.init (); m_core.open_sync (filename); @@ -724,8 +700,7 @@ bool basic_fstream_dyad<_CharT, _Traits>::is_open () } #else //----------------------------------------------------------------------- template -basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad ( - const dyad_stream_core& core) +basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (const dyad_stream_core& core) : m_core (core) { m_stream = std::unique_ptr (new basic_fstream ()); @@ -739,36 +714,31 @@ basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad () } template -basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad ( - const char* filename, - std::ios_base::openmode mode) +basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (const char* filename, + std::ios_base::openmode mode) { m_core.init (); m_core.open_sync (filename); - m_stream = - std::unique_ptr (new basic_fstream (filename, mode)); + m_stream = std::unique_ptr (new basic_fstream (filename, mode)); if ((m_stream != nullptr) && (*m_stream)) { m_filename = std::string{filename}; } } template -basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad ( - const string& filename, - std::ios_base::openmode mode) +basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (const string& filename, + std::ios_base::openmode mode) { m_core.init (); m_core.open_sync (filename.c_str ()); - m_stream = - std::unique_ptr (new basic_fstream (filename, mode)); + m_stream = std::unique_ptr (new basic_fstream (filename, mode)); if ((m_stream != nullptr) && (*m_stream)) { m_filename = filename; } } template -basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad ( - basic_fstream_dyad&& rhs) +basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (basic_fstream_dyad&& rhs) : m_stream (std::move (rhs.m_stream)), m_core (std::move (rhs.m_core)) { } @@ -776,20 +746,19 @@ basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad ( #if (__cplusplus >= 201703L) && __has_include() template template -basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad ( - const _Path& filepath, - std::ios_base::openmode mode) +basic_fstream_dyad<_CharT, _Traits>::basic_fstream_dyad (const _Path& filepath, + std::ios_base::openmode mode) { m_core.init (); m_core.open_sync (filepath.c_str ()); - m_stream = std::unique_ptr ( - new basic_fstream (filepath.c_str (), mode)); + m_stream = + std::unique_ptr (new basic_fstream (filepath.c_str (), mode)); } #endif // c++17 filesystem template -basic_fstream_dyad<_CharT, _Traits>& basic_fstream_dyad<_CharT, _Traits>:: -operator= (basic_fstream_dyad&& rhs) +basic_fstream_dyad<_CharT, _Traits>& basic_fstream_dyad<_CharT, _Traits>::operator= ( + basic_fstream_dyad&& rhs) { m_stream = std::move (rhs.m_stream); m_core = std::move (rhs.m_core); @@ -881,8 +850,7 @@ std::filebuf* basic_fstream_dyad<_CharT, _Traits>::rdbuf () const } template -std::basic_fstream<_CharT, _Traits>& basic_fstream_dyad<_CharT, - _Traits>::get_stream () +std::basic_fstream<_CharT, _Traits>& basic_fstream_dyad<_CharT, _Traits>::get_stream () { if (m_stream == nullptr) { // TODO: throw diff --git a/src/stream/dyad_stream_core.cpp b/src/stream/dyad_stream_core.cpp index 3ea6eea1..9ccabce9 100644 --- a/src/stream/dyad_stream_core.cpp +++ b/src/stream/dyad_stream_core.cpp @@ -8,8 +8,9 @@ * SPDX-License-Identifier: LGPL-3.0 \************************************************************/ -#include "dyad_core.h" #include "dyad_stream_core.hpp" + +#include "dyad_core.h" #include "murmur3.h" #include "utils.h" @@ -26,7 +27,7 @@ #include #include using namespace std; // std::clock () -//#include // c++11 +// #include // c++11 #include #include @@ -64,67 +65,22 @@ void dyad_stream_core::init () { char *e = NULL; - bool debug = false; - bool check = false; - bool shared_storage = false; - unsigned int key_depth = 0; - unsigned int key_bins = 0; - const char *kvs_namespace = NULL; - const char *cons_managed_path = NULL; - const char *prod_managed_path = NULL; - if (m_initialized) { return; } - if ((e = getenv (DYAD_SYNC_DEBUG_ENV)) && (atoi (e) != 0)) { - debug = true; - enable_debug_dyad_utils (); - fprintf (stderr, "DYAD_WRAPPER: Initializeing DYAD wrapper\n"); - } else { - debug = false; - disable_debug_dyad_utils (); - } - - if ((e = getenv (DYAD_SHARED_STORAGE_ENV)) && (atoi (e) != 0)) - shared_storage = true; - else - shared_storage = false; - - if ((e = getenv (DYAD_KEY_DEPTH_ENV))) - key_depth = atoi (e); - else - key_depth = 2; - - if ((e = getenv (DYAD_KEY_BINS_ENV))) - key_bins = atoi (e); - else - key_bins = 256; - - if ((e = getenv (DYAD_KVS_NAMESPACE_ENV))) { - kvs_namespace = e; - } else { - kvs_namespace = NULL; - } - if ((e = getenv (DYAD_PATH_CONSUMER_ENV))) { - cons_managed_path = e; - m_is_cons = (strlen (cons_managed_path) != 0); + m_is_cons = (strlen (e) != 0); } else { - cons_managed_path = NULL; m_is_cons = false; } if ((e = getenv (DYAD_PATH_PRODUCER_ENV))) { - prod_managed_path = e; - m_is_prod = (strlen (prod_managed_path) != 0); + m_is_prod = (strlen (e) != 0); } else { - prod_managed_path = NULL; m_is_prod = false; } - dyad_rc_t rc = - dyad_init (debug, check, shared_storage, key_depth, key_bins, - kvs_namespace, prod_managed_path, cons_managed_path, &m_ctx); + dyad_rc_t rc = dyad_init_env (&m_ctx); // TODO figure out if we want to error if init fails m_initialized = true; @@ -134,11 +90,16 @@ void dyad_stream_core::init () void dyad_stream_core::init (const dyad_params &p) { DPRINTF (m_ctx, "DYAD_WRAPPER: Initializeing DYAD wrapper\n"); - dyad_rc_t rc = - dyad_init (p.m_debug, false, p.m_shared_storage, p.m_key_depth, - p.m_key_bins, p.m_kvs_namespace.c_str (), - p.m_prod_managed_path.c_str (), - p.m_cons_managed_path.c_str (), &m_ctx); + dyad_rc_t rc = dyad_init (p.m_debug, + false, + p.m_shared_storage, + p.m_key_depth, + p.m_key_bins, + p.m_kvs_namespace.c_str (), + p.m_prod_managed_path.c_str (), + p.m_cons_managed_path.c_str (), + static_cast (p.m_dtl_mode), + &m_ctx); // TODO figure out if we want to error if init fails m_initialized = true; log_info ("Stream core is initialized by parameters"); @@ -147,18 +108,19 @@ void dyad_stream_core::init (const dyad_params &p) void dyad_stream_core::log_info (const std::string &msg_head) const { DYAD_LOG_INFO (m_ctx, "=== %s ===\n", msg_head.c_str ()); - DYAD_LOG_INFO (m_ctx, "%s=%s\n", DYAD_PATH_CONSUMER_ENV, - m_ctx->cons_managed_path); - DYAD_LOG_INFO (m_ctx, "%s=%s\n", DYAD_PATH_PRODUCER_ENV, - m_ctx->prod_managed_path); - DYAD_LOG_INFO (m_ctx, "%s=%s\n", DYAD_SYNC_DEBUG_ENV, + DYAD_LOG_INFO (m_ctx, "%s=%s\n", DYAD_PATH_CONSUMER_ENV, m_ctx->cons_managed_path); + DYAD_LOG_INFO (m_ctx, "%s=%s\n", DYAD_PATH_PRODUCER_ENV, m_ctx->prod_managed_path); + DYAD_LOG_INFO (m_ctx, + "%s=%s\n", + DYAD_SYNC_DEBUG_ENV, (m_ctx->debug) ? "true" : "false"); - DYAD_LOG_INFO (m_ctx, "%s=%s\n", DYAD_SHARED_STORAGE_ENV, + DYAD_LOG_INFO (m_ctx, + "%s=%s\n", + DYAD_SHARED_STORAGE_ENV, (m_ctx->shared_storage) ? "true" : "false"); DYAD_LOG_INFO (m_ctx, "%s=%u\n", DYAD_KEY_DEPTH_ENV, m_ctx->key_depth); DYAD_LOG_INFO (m_ctx, "%s=%u\n", DYAD_KEY_BINS_ENV, m_ctx->key_bins); - DYAD_LOG_INFO (m_ctx, "%s=%s\n", DYAD_KVS_NAMESPACE_ENV, - m_ctx->kvs_namespace); + DYAD_LOG_INFO (m_ctx, "%s=%s\n", DYAD_KVS_NAMESPACE_ENV, m_ctx->kvs_namespace); } bool dyad_stream_core::is_dyad_producer () diff --git a/src/urpc/Makefile.am b/src/urpc/Makefile.am index 10fa200d..46e1958c 100644 --- a/src/urpc/Makefile.am +++ b/src/urpc/Makefile.am @@ -1,8 +1,10 @@ lib_LTLIBRARIES = liburpc_client.la liburpc_client_la_SOURCES = urpc_client.c urpc_client.hpp liburpc_client_la_LIBADD = $(FLUX_CORE_LIBS) $(top_builddir)/src/utils/libutils.la -liburpc_client_la_CPPFLAGS = $(FLUX_CORE_CFLAGS) -DURPC_CHECK=1 -I$(top_builddir)/src/utils +liburpc_client_la_CFLAGS = $(FLUX_CORE_CFLAGS) -DURPC_CHECK=1 -I$(top_builddir)/src/utils +liburpc_client_la_CPPFLAGS = if PERFFLOW liburpc_client_la_LIBADD += $(PERFFLOW_LIBS) -liburpc_client_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS) $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1 +liburpc_client_la_CFLAGS += $(PERFFLOW_CFLAGS) -DDYAD_PERFFLOW=1 +liburpc_client_la_CPPFLAGS += $(PERFFLOW_PLUGIN_CPPFLAGS) endif diff --git a/src/urpc/urpc_client.c b/src/urpc/urpc_client.c index 317e9b79..4b010bca 100644 --- a/src/urpc/urpc_client.c +++ b/src/urpc/urpc_client.c @@ -21,7 +21,7 @@ #include #include using namespace std; // std::clock () -//#include // c++11 +// #include // c++11 #else #include #include @@ -109,8 +109,7 @@ static int urpc_rpc_pack (flux_future_t **ft, uint32_t rank, const char *cmd) { // Send the request to execute a command - if (!(*ft = flux_rpc_pack (ctx->h, "urpc.exec", rank, 0, "{s:s}", "cmd", - cmd))) { + if (!(*ft = flux_rpc_pack (ctx->h, "urpc.exec", rank, 0, "{s:s}", "cmd", cmd))) { FLUX_LOG_ERR ("flux_rpc_pack({urpc.exec %s})", cmd); return -1; } @@ -182,8 +181,7 @@ int urpc_client (uint32_t server_rank, const char *cmd, const char *file_path, i // TODO: Need to be consistent with the mode at the source odir = dirname (file_path_copy); m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); - if ((strncmp (odir, ".", strlen (".")) != 0) - && (mkdir_as_needed (odir, m) < 0)) { + if ((strncmp (odir, ".", strlen (".")) != 0) && (mkdir_as_needed (odir, m) < 0)) { FLUX_LOG_ERR ("Failed to create directory \"%s\".\n", odir); goto done; } diff --git a/src/urpc/urpc_client.h b/src/urpc/urpc_client.h index cfd9e54a..450b449b 100644 --- a/src/urpc/urpc_client.h +++ b/src/urpc/urpc_client.h @@ -8,8 +8,8 @@ * SPDX-License-Identifier: LGPL-3.0 \************************************************************/ -#ifndef URPC_CLIENT_H -#define URPC_CLIENT_H +#ifndef DYAD_URPC_URPC_CLIENT_H +#define DYAD_URPC_URPC_CLIENT_H #ifdef __cplusplus extern "C" { @@ -26,7 +26,7 @@ int urpc_client (uint32_t server_rank, } #endif // __cplusplus -#endif /* URPC_CLIENT_H */ +#endif /* DYAD_URPC_URPC_CLIENT_H */ /* * vi: ts=4 sw=4 expandtab diff --git a/src/urpc/urpc_ctx.h b/src/urpc/urpc_ctx.h index ee115ca4..c28c5077 100644 --- a/src/urpc/urpc_ctx.h +++ b/src/urpc/urpc_ctx.h @@ -8,8 +8,8 @@ * SPDX-License-Identifier: LGPL-3.0 \************************************************************/ -#ifndef URPC_CLIENT_CTX_H -#define URPC_CLIENT_CTX_H +#ifndef DYAD_URPC_URPC_CLIENT_CTX_H +#define DYAD_URPC_URPC_CLIENT_CTX_H #ifdef __cplusplus #include @@ -44,4 +44,4 @@ inline void init_urpc_client_ctx (urpc_client_ctx_t *ctx) } #endif // __cplusplus -#endif // URPC_CLIENT_CTX_H +#endif // DYAD_URPC_URPC_CLIENT_CTX_H diff --git a/src/utils/Makefile.am b/src/utils/Makefile.am index 3fd842ad..ed069e66 100644 --- a/src/utils/Makefile.am +++ b/src/utils/Makefile.am @@ -1,9 +1,17 @@ -SUBDIRS = libtap +SUBDIRS = libtap base64 noinst_LTLIBRARIES = libutils.la libmurmur3.la -libutils_la_SOURCES = utils.c utils.h dyad.h -libutils_la_CPPFLAGS = $(AM_CPPFLAGS) $(FLUX_CORE_CFLAGS) -#libutils_la_LIBADD = +libutils_la_SOURCES = utils.c utils.h read_all.c read_all.h +libutils_la_CFLAGS = \ + $(AM_CFLAGS) \ + $(FLUX_CORE_CFLAGS) \ + -fvisibility=hidden +libutils_la_LIBADD = \ + $(top_builddir)/src/utils/base64/libbase64.la \ + $(FLUX_CORE_LIBS) libmurmur3_la_SOURCES = murmur3.c murmur3.h +libmurmur3_la_CFLAGS = \ + $(AM_CFLAGS) \ + -fvisibility=hidden #libmurmur3_la_CPPFLAGS = $(AM_CPPFLAGS) #libmurmur3_la_LIBADD = diff --git a/src/utils/base64/Makefile.am b/src/utils/base64/Makefile.am new file mode 100644 index 00000000..fde9ddaf --- /dev/null +++ b/src/utils/base64/Makefile.am @@ -0,0 +1,6 @@ +noinst_LTLIBRARIES = libbase64.la + +libbase64_la_SOURCES = base64.c base64.h +libbase64_la_CFLAGS = \ + $(AM_CFLAGS) \ + -fvisibility=hidden diff --git a/src/utils/base64/base64.c b/src/utils/base64/base64.c new file mode 100644 index 00000000..5fa7f114 --- /dev/null +++ b/src/utils/base64/base64.c @@ -0,0 +1,260 @@ +/* Licensed under BSD-MIT - see LICENSE file for details */ +#include "base64.h" + +#include +#include +#include +#include + +/** + * sixbit_to_b64 - maps a 6-bit value to the base64 alphabet + * @param map A base 64 map (see base64_init_map) + * @param sixbit Six-bit value to map + * @return a base 64 character + */ +static char sixbit_to_b64 (const base64_maps_t *maps, const uint8_t sixbit) +{ + assert (sixbit <= 63); + + return maps->encode_map[(unsigned char)sixbit]; +} + +/** + * sixbit_from_b64 - maps a base64-alphabet character to its 6-bit value + * @param maps A base 64 maps structure (see base64_init_maps) + * @param sixbit Six-bit value to map + * @return a six-bit value + */ +static int8_t sixbit_from_b64 (const base64_maps_t *maps, const unsigned char b64letter) +{ + int8_t ret; + + ret = maps->decode_map[(unsigned char)b64letter]; + if (ret == (int8_t)0xff) { + errno = EDOM; + return -1; + } + + return ret; +} + +bool base64_char_in_alphabet (const base64_maps_t *maps, const char b64char) +{ + return (maps->decode_map[(const unsigned char)b64char] != (int8_t)0xff); +} + +void base64_init_maps (base64_maps_t *dest, const char src[64]) +{ + unsigned char i; + + memcpy (dest->encode_map, src, 64); + memset (dest->decode_map, 0xff, 256); + for (i = 0; i < 64; i++) { + dest->decode_map[(unsigned char)src[i]] = i; + } +} + +size_t base64_encoded_length (size_t srclen) +{ + return ((srclen + 2) / 3) * 4; +} + +void base64_encode_triplet_using_maps (const base64_maps_t *maps, + char dest[4], + const char src[3]) +{ + char a = src[0]; + char b = src[1]; + char c = src[2]; + + dest[0] = sixbit_to_b64 (maps, (a & 0xfc) >> 2); + dest[1] = sixbit_to_b64 (maps, ((a & 0x3) << 4) | ((b & 0xf0) >> 4)); + dest[2] = sixbit_to_b64 (maps, ((c & 0xc0) >> 6) | ((b & 0xf) << 2)); + dest[3] = sixbit_to_b64 (maps, c & 0x3f); +} + +void base64_encode_tail_using_maps (const base64_maps_t *maps, + char dest[4], + const char *src, + const size_t srclen) +{ + char longsrc[3] = {0}; + + assert (srclen <= 3); + + memcpy (longsrc, src, srclen); + base64_encode_triplet_using_maps (maps, dest, longsrc); + memset (dest + 1 + srclen, '=', 3 - srclen); +} + +ssize_t base64_encode_using_maps (const base64_maps_t *maps, + char *dest, + const size_t destlen, + const char *src, + const size_t srclen) +{ + size_t src_offset = 0; + size_t dest_offset = 0; + + if (destlen < base64_encoded_length (srclen)) { + errno = EOVERFLOW; + return -1; + } + + while (srclen - src_offset >= 3) { + base64_encode_triplet_using_maps (maps, &dest[dest_offset], &src[src_offset]); + src_offset += 3; + dest_offset += 4; + } + + if (src_offset < srclen) { + base64_encode_tail_using_maps (maps, + &dest[dest_offset], + &src[src_offset], + srclen - src_offset); + dest_offset += 4; + } + + memset (&dest[dest_offset], '\0', destlen - dest_offset); + + return dest_offset; +} + +size_t base64_decoded_length (size_t srclen) +{ + return ((srclen + 3) / 4 * 3); +} + +ssize_t base64_decode_quartet_using_maps (const base64_maps_t *maps, + char dest[3], + const char src[4]) +{ + signed char a; + signed char b; + signed char c; + signed char d; + + a = sixbit_from_b64 (maps, src[0]); + b = sixbit_from_b64 (maps, src[1]); + c = sixbit_from_b64 (maps, src[2]); + d = sixbit_from_b64 (maps, src[3]); + + if ((a == -1) || (b == -1) || (c == -1) || (d == -1)) { + return -1; + } + + dest[0] = (a << 2) | (b >> 4); + dest[1] = ((b & 0xf) << 4) | (c >> 2); + dest[2] = ((c & 0x3) << 6) | d; + + return 0; +} + +ssize_t base64_decode_tail_using_maps (const base64_maps_t *maps, + char dest[3], + const char *src, + const size_t srclen) +{ + char longsrc[4]; + int quartet_result; + size_t insize = srclen; + + while (insize != 0 && src[insize - 1] == '=') { /* throw away padding symbols */ + insize--; + } + if (insize == 0) { + return 0; + } + if (insize == 1) { + /* the input is malformed.... */ + errno = EINVAL; + return -1; + } + memcpy (longsrc, src, insize); + memset (longsrc + insize, 'A', 4 - insize); + quartet_result = base64_decode_quartet_using_maps (maps, dest, longsrc); + if (quartet_result == -1) { + return -1; + } + + return insize - 1; +} + +ssize_t base64_decode_using_maps (const base64_maps_t *maps, + char *dest, + const size_t destlen, + const char *src, + const size_t srclen) +{ + ssize_t dest_offset = 0; + ssize_t i; + ssize_t more; + + if (destlen < base64_decoded_length (srclen)) { + errno = EOVERFLOW; + return -1; + } + + for (i = 0; srclen - i > 4; i += 4) { + if (base64_decode_quartet_using_maps (maps, &dest[dest_offset], &src[i]) == -1) { + return -1; + } + dest_offset += 3; + } + + more = base64_decode_tail_using_maps (maps, &dest[dest_offset], &src[i], srclen - i); + if (more == -1) { + return -1; + } + dest_offset += more; + + memset (&dest[dest_offset], '\0', destlen - dest_offset); + + return dest_offset; +} + +/** + * base64_maps_rfc4648 - pregenerated maps struct for rfc4648 + */ +const base64_maps_t base64_maps_rfc4648 = { + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", + + "\xff\xff\xff\xff\xff" /* 0 */ + "\xff\xff\xff\xff\xff" /* 5 */ + "\xff\xff\xff\xff\xff" /* 10 */ + "\xff\xff\xff\xff\xff" /* 15 */ + "\xff\xff\xff\xff\xff" /* 20 */ + "\xff\xff\xff\xff\xff" /* 25 */ + "\xff\xff\xff\xff\xff" /* 30 */ + "\xff\xff\xff\xff\xff" /* 35 */ + "\xff\xff\xff\x3e\xff" /* 40 */ + "\xff\xff\x3f\x34\x35" /* 45 */ + "\x36\x37\x38\x39\x3a" /* 50 */ + "\x3b\x3c\x3d\xff\xff" /* 55 */ + "\xff\xff\xff\xff\xff" /* 60 */ + "\x00\x01\x02\x03\x04" /* 65 A */ + "\x05\x06\x07\x08\x09" /* 70 */ + "\x0a\x0b\x0c\x0d\x0e" /* 75 */ + "\x0f\x10\x11\x12\x13" /* 80 */ + "\x14\x15\x16\x17\x18" /* 85 */ + "\x19\xff\xff\xff\xff" /* 90 */ + "\xff\xff\x1a\x1b\x1c" /* 95 */ + "\x1d\x1e\x1f\x20\x21" /* 100 */ + "\x22\x23\x24\x25\x26" /* 105 */ + "\x27\x28\x29\x2a\x2b" /* 110 */ + "\x2c\x2d\x2e\x2f\x30" /* 115 */ + "\x31\x32\x33\xff\xff" /* 120 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 125 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 155 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 185 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 215 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 245 */ +}; diff --git a/src/utils/base64/base64.h b/src/utils/base64/base64.h new file mode 100644 index 00000000..40c27533 --- /dev/null +++ b/src/utils/base64/base64.h @@ -0,0 +1,257 @@ +/* Licensed under BSD-MIT - see LICENSE file for details */ +#ifndef CCAN_BASE64_H +#define CCAN_BASE64_H + +#ifdef __cplusplus +#include +#else +#include +#include +#endif + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * base64_maps_t - structure to hold maps for encode/decode + */ +typedef struct { + char encode_map[64]; + signed char decode_map[256]; +} base64_maps_t; + +/** + * base64_encoded_length - Calculate encode buffer length + * @param srclen the size of the data to be encoded + * @note add 1 to this to get null-termination + * @return Buffer length required for encode + */ +size_t base64_encoded_length (size_t srclen); + +/** + * base64_decoded_length - Calculate decode buffer length + * @param srclen Length of the data to be decoded + * @note This does not return the size of the decoded data! see base64_decode + * @return Minimum buffer length for safe decode + */ +size_t base64_decoded_length (size_t srclen); + +/** + * base64_init_maps - populate a base64_maps_t based on a supplied alphabet + * @param dest A base64 maps object + * @param src Alphabet to populate the maps from (e.g. base64_alphabet_rfc4648) + */ +void base64_init_maps (base64_maps_t *dest, const char src[64]); + +/** + * base64_encode_triplet_using_maps - encode 3 bytes into base64 using a specific + * alphabet + * @param maps Maps to use for encoding (see base64_init_maps) + * @param dest Buffer containing 3 bytes + * @param src Buffer containing 4 characters + */ +void base64_encode_triplet_using_maps (const base64_maps_t *maps, + char dest[4], + const char src[3]); + +/** + * base64_encode_tail_using_maps - encode the final bytes of a source using a specific + * alphabet + * @param maps Maps to use for encoding (see base64_init_maps) + * @param dest Buffer containing 4 bytes + * @param src Buffer containing srclen bytes + * @param srclen Number of bytes (<= 3) to encode in src + */ +void base64_encode_tail_using_maps (const base64_maps_t *maps, + char dest[4], + const char *src, + size_t srclen); + +/** + * base64_encode_using_maps - encode a buffer into base64 using a specific alphabet + * @param maps Maps to use for encoding (see base64_init_maps) + * @param dest Buffer to encode into + * @param destlen Length of dest + * @param src Buffer to encode + * @param srclen Length of the data to encode + * @return Number of encoded bytes set in dest. -1 on error (and errno set) + * @note dest will be nul-padded to destlen (past any required padding) + * @note sets errno = EOVERFLOW if destlen is too small + */ +ssize_t base64_encode_using_maps (const base64_maps_t *maps, + char *dest, + size_t destlen, + const char *src, + size_t srclen); + +/* + * base64_char_in_alphabet - returns true if character can be part of an encoded string + * @param maps A base64 maps object (see base64_init_maps) + * @param b64char Character to check + */ +bool base64_char_in_alphabet (const base64_maps_t *maps, char b64char); + +/** + * base64_decode_using_maps - decode a base64-encoded string using a specific alphabet + * @param maps A base64 maps object (see base64_init_maps) + * @param dest Buffer to decode into + * @param destlen length of dest + * @param src the buffer to decode + * @param srclen the length of the data to decode + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note dest will be nul-padded to destlen + * @note sets errno = EOVERFLOW if destlen is too small + * @note sets errno = EDOM if src contains invalid characters + */ +ssize_t base64_decode_using_maps (const base64_maps_t *maps, + char *dest, + size_t destlen, + const char *src, + size_t srclen); + +/** + * base64_decode_quartet_using_maps - decode 4 bytes from base64 using a specific + * alphabet + * @param maps A base64 maps object (see base64_init_maps) + * @param dest Buffer containing 3 bytes + * @param src Buffer containing 4 bytes + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note sets errno = EDOM if src contains invalid characters + */ +ssize_t base64_decode_quartet_using_maps (const base64_maps_t *maps, + char dest[3], + const char src[4]); + +/** + * base64_decode_tail_using_maps - decode the final bytes of a base64 string using a + * specific alphabet + * @param maps A base64 maps object (see base64_init_maps) + * @param dest Buffer containing 3 bytes + * @param src Buffer containing 4 bytes - padded with '=' as required + * @param srclen Number of bytes to decode in src + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note sets errno = EDOM if src contains invalid characters + * @note sets errno = EINVAL if src is an invalid base64 tail + */ +ssize_t base64_decode_tail_using_maps (const base64_maps_t *maps, + char dest[3], + const char *src, + size_t srclen); + +/* the rfc4648 functions: */ + +extern const base64_maps_t base64_maps_rfc4648; + +/** + * base64_encode - Encode a buffer into base64 according to rfc4648 + * @param dest Buffer to encode into + * @param destlen Length of the destination buffer + * @param src Buffer to encode + * @param srclen Length of the data to encode + * @return Number of encoded bytes set in dest. -1 on error (and errno set) + * @note dest will be nul-padded to destlen (past any required padding) + * @note sets errno = EOVERFLOW if destlen is too small + * + * This function encodes src according to http://tools.ietf.org/html/rfc4648 + * + * Example: + * size_t encoded_length; + * char dest[100]; + * const char *src = "This string gets encoded"; + * encoded_length = base64_encode(dest, sizeof(dest), src, strlen(src)); + * printf("Returned data of length %zd @%p\n", encoded_length, &dest); + */ +static inline ssize_t base64_encode (char *dest, + size_t destlen, + const char *src, + size_t srclen) +{ + return base64_encode_using_maps (&base64_maps_rfc4648, dest, destlen, src, srclen); +} + +/** + * base64_encode_triplet - encode 3 bytes into base64 according to rfc4648 + * @param dest Buffer containing 4 bytes + * @param src Buffer containing 3 bytes + */ +static inline void base64_encode_triplet (char dest[4], const char src[3]) +{ + base64_encode_triplet_using_maps (&base64_maps_rfc4648, dest, src); +} + +/** + * base64_encode_tail - encode the final bytes of a source according to rfc4648 + * @param dest Buffer containing 4 bytes + * @param src Buffer containing srclen bytes + * @param srclen Number of bytes (<= 3) to encode in src + */ +static inline void base64_encode_tail (char dest[4], const char *src, size_t srclen) +{ + base64_encode_tail_using_maps (&base64_maps_rfc4648, dest, src, srclen); +} + +/** + * base64_decode - decode An rfc4648 base64-encoded string + * @param dest Buffer to decode into + * @param destlen Length of the destination buffer + * @param src Buffer to decode + * @param srclen Length of the data to decode + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note dest will be nul-padded to destlen + * @note sets errno = EOVERFLOW if destlen is too small + * @note sets errno = EDOM if src contains invalid characters + * + * This function decodes the buffer according to + * http://tools.ietf.org/html/rfc4648 + * + * Example: + * size_t decoded_length; + * char ret[100]; + * const char *src = "Zm9vYmFyYmF6"; + * decoded_length = base64_decode(ret, sizeof(ret), src, strlen(src)); + * printf("Returned data of length %zd @%p\n", decoded_length, &ret); + */ +static inline ssize_t base64_decode (char *dest, + size_t destlen, + const char *src, + size_t srclen) +{ + return base64_decode_using_maps (&base64_maps_rfc4648, dest, destlen, src, srclen); +} + +/** + * base64_decode_quartet - decode the first 4 characters in src into dest + * @param dest Buffer containing 3 bytes + * @param src Buffer containing 4 characters + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note sets errno = EDOM if src contains invalid characters + */ +static inline ssize_t base64_decode_quartet (char dest[3], const char src[4]) +{ + return base64_decode_quartet_using_maps (&base64_maps_rfc4648, dest, src); +} + +/** + * @brief decode the final bytes of a base64 string from src into dest + * @param dest Buffer containing 3 bytes + * @param src Buffer containing 4 bytes - padded with '=' as required + * @param srclen Number of bytes to decode in src + * @return Number of decoded bytes set in dest. -1 on error (and errno set) + * @note sets errno = EDOM if src contains invalid characters + * @note sets errno = EINVAL if src is an invalid base64 tail + */ +static inline ssize_t base64_decode_tail (char dest[3], const char *src, size_t srclen) +{ + return base64_decode_tail_using_maps (&base64_maps_rfc4648, dest, src, srclen); +} + +/* end rfc4648 functions */ + +#ifdef __cplusplus +} +#endif + +#endif /* CCAN_BASE64_H */ diff --git a/src/utils/base64/license b/src/utils/base64/license new file mode 100644 index 00000000..89de3547 --- /dev/null +++ b/src/utils/base64/license @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/src/utils/libtap/tap.c b/src/utils/libtap/tap.c index 839116a5..cc51b85f 100644 --- a/src/utils/libtap/tap.c +++ b/src/utils/libtap/tap.c @@ -55,11 +55,7 @@ void tap_plan (int tests, const char *fmt, ...) } } -int vok_at_loc (const char *file, - int line, - int test, - const char *fmt, - va_list args) +int vok_at_loc (const char *file, int line, int test, const char *fmt, va_list args) { char *name = vstrdupf (fmt, args); if (!test) { @@ -183,10 +179,7 @@ int cmp_ok_at_loc (const char *file, return test; } -static int find_mem_diff (const char *a, - const char *b, - size_t n, - size_t *offset) +static int find_mem_diff (const char *a, const char *b, size_t n, size_t *offset) { size_t i; if (a == b) @@ -259,13 +252,17 @@ int exit_status () if (expected_tests == NO_PLAN) { printf ("1..%d\n", current_test); } else if (current_test != expected_tests) { - diag ("Looks like you planned %d test%s but ran %d.", expected_tests, - expected_tests > 1 ? "s" : "", current_test); + diag ("Looks like you planned %d test%s but ran %d.", + expected_tests, + expected_tests > 1 ? "s" : "", + current_test); retval = 2; } if (failed_tests) { - diag ("Looks like you failed %d test%s of %d run.", failed_tests, - failed_tests > 1 ? "s" : "", current_test); + diag ("Looks like you failed %d test%s of %d run.", + failed_tests, + failed_tests > 1 ? "s" : "", + current_test); retval = 1; } return retval; @@ -327,8 +324,12 @@ int tap_test_died (int status) static int *test_died = NULL; int prev; if (!test_died) { - test_died = mmap (0, sizeof (int), PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, -1, 0); + test_died = mmap (0, + sizeof (int), + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, + -1, + 0); *test_died = 0; } prev = *test_died; @@ -351,8 +352,12 @@ int like_at_loc (int for_match, if (err) { char errbuf[256]; regerror (err, &re, errbuf, sizeof errbuf); - fprintf (stderr, "Unable to compile regex '%s': %s at %s line %d\n", - expected, errbuf, file, line); + fprintf (stderr, + "Unable to compile regex '%s': %s at %s line %d\n", + expected, + errbuf, + file, + line); exit (255); } err = regexec (&re, got, 0, NULL, 0); diff --git a/src/utils/libtap/tap.h b/src/utils/libtap/tap.h index 06facca4..de982458 100644 --- a/src/utils/libtap/tap.h +++ b/src/utils/libtap/tap.h @@ -23,11 +23,7 @@ extern "C" { #include #include -int vok_at_loc (const char *file, - int line, - int test, - const char *fmt, - va_list args); +int vok_at_loc (const char *file, int line, int test, const char *fmt, va_list args); int ok_at_loc (const char *file, int line, int test, const char *fmt, ...); int is_at_loc (const char *file, int line, @@ -95,8 +91,7 @@ void tap_end_todo (void); #ifdef _WIN32 #define like(...) tap_skip (1, "like is not implemented on Windows") #define unlike tap_skip (1, "unlike is not implemented on Windows") -#define dies_ok_common(...) \ - tap_skip (1, "Death detection is not supported on Windows") +#define dies_ok_common(...) tap_skip (1, "Death detection is not supported on Windows") #else #define like(...) like_at_loc (1, __FILE__, __LINE__, __VA_ARGS__, NULL) #define unlike(...) like_at_loc (0, __FILE__, __LINE__, __VA_ARGS__, NULL) diff --git a/src/utils/murmur3.c b/src/utils/murmur3.c index 82b9411b..77559e64 100644 --- a/src/utils/murmur3.c +++ b/src/utils/murmur3.c @@ -128,10 +128,7 @@ void MurmurHash3_x86_32 (const void *key, int len, uint32_t seed, void *out) //----------------------------------------------------------------------------- -void MurmurHash3_x86_128 (const void *key, - const int len, - uint32_t seed, - void *out) +void MurmurHash3_x86_128 (const void *key, const int len, uint32_t seed, void *out) { const uint8_t *data = (const uint8_t *)key; const int nblocks = len / 16; @@ -292,10 +289,7 @@ void MurmurHash3_x86_128 (const void *key, //----------------------------------------------------------------------------- -void MurmurHash3_x64_128 (const void *key, - const int len, - const uint32_t seed, - void *out) +void MurmurHash3_x64_128 (const void *key, const int len, const uint32_t seed, void *out) { const uint8_t *data = (const uint8_t *)key; const int nblocks = len / 16; diff --git a/src/modules/read_all.c b/src/utils/read_all.c similarity index 100% rename from src/modules/read_all.c rename to src/utils/read_all.c diff --git a/src/modules/read_all.h b/src/utils/read_all.h similarity index 87% rename from src/modules/read_all.h rename to src/utils/read_all.h index 5564fd01..23a0ed78 100644 --- a/src/modules/read_all.h +++ b/src/utils/read_all.h @@ -8,8 +8,8 @@ * SPDX-License-Identifier: LGPL-3.0 \************************************************************/ -#ifndef DYAD_MODULES_READ_ALL_H -#define DYAD_MODULES_READ_ALL_H +#ifndef DYAD_UTILS_READ_ALL_H +#define DYAD_UTILS_READ_ALL_H #include @@ -29,4 +29,4 @@ read_all (int fd, void **bufp); }; #endif // defined(__cplusplus) -#endif /* !DYAD_MODULES_READ_ALL_H */ +#endif /* DYAD_UTILS_READ_ALL_H */ diff --git a/src/utils/utils.c b/src/utils/utils.c index d8fd80aa..83b104e7 100644 --- a/src/utils/utils.c +++ b/src/utils/utils.c @@ -26,7 +26,7 @@ #include #include #include -//#include // c++11 +// #include // c++11 #else #include #include // PATH_MAX @@ -94,8 +94,7 @@ char* concat_str (char* __restrict__ str, bool con_end = false; if ((connector != NULL) && (str_len_org >= con_len)) { - con_end = - (strncmp (str + str_len_org - con_len, connector, con_len) == 0); + con_end = (strncmp (str + str_len_org - con_len, connector, con_len) == 0); } const size_t str_len = (con_end ? (str_len_org - con_len) : str_len_org); @@ -160,11 +159,9 @@ bool cmp_prefix (const char* __restrict__ prefix, { const char* const u_len_end = ((const char*)u_len) + sizeof (size_t); bool no_overlap = - ((prefix + strlen (prefix) <= (char*)u_len) - || (u_len_end <= prefix)) + ((prefix + strlen (prefix) <= (char*)u_len) || (u_len_end <= prefix)) && ((full + strlen (full) <= (char*)u_len) || (u_len_end <= full)) - && ((delim + strlen (delim) <= (char*)u_len) - || (u_len_end <= delim)); + && ((delim + strlen (delim) <= (char*)u_len) || (u_len_end <= delim)); if (!no_overlap) { DPRINTF ("DYAD UTIL: buffers overlap.\n"); @@ -232,9 +229,8 @@ bool cmp_canonical_path_prefix (const char* __restrict__ prefix, { { const char* const upath_end = upath + upath_capacity; - bool no_overlap = - ((prefix + strlen (prefix) <= upath) || (upath_end <= prefix)) - && ((path + strlen (path) <= upath) || (upath_end <= path)); + bool no_overlap = ((prefix + strlen (prefix) <= upath) || (upath_end <= prefix)) + && ((path + strlen (path) <= upath) || (upath_end <= path)); if (!no_overlap) { DPRINTF ("DYAD UTIL: buffers overlap\n"); @@ -321,7 +317,9 @@ int mkdir_as_needed (const char* path, const mode_t m) "Directory \"%s\" already exists with " "different permission bits %o from " "the requested %o\n", - path, (sb.st_mode & RWX_UGO), (m & RWX_UGO)); + path, + (sb.st_mode & RWX_UGO), + (m & RWX_UGO)); return 5; // already exists but with different mode } return 1; // already exists @@ -339,13 +337,14 @@ int mkdir_as_needed (const char* path, const mode_t m) "Directory \"%s\" already exists with " "different permission bits %o from " "the requested %o\n", - path, (sb.st_mode & RWX_UGO), (m & RWX_UGO)); + path, + (sb.st_mode & RWX_UGO), + (m & RWX_UGO)); return 5; // already exists but with different mode } return 1; // already exists } - DPRINTF ("Cannot create directory \"%s\": %s\n", path, - strerror (errno)); + DPRINTF ("Cannot create directory \"%s\": %s\n", path, strerror (errno)); perror ("mkdir_as_needed() "); return -1; } @@ -371,11 +370,11 @@ int get_path (const int fd, const size_t max_size, char* path) ssize_t rc = readlink (proclink, path, max_size); if (rc < (ssize_t)0) { IPRINTF ("DYAD UTIL: error reading the file link (%s): %s\n", - strerror (errno), proclink); + strerror (errno), + proclink); return -1; } else if ((size_t)rc == max_size) { - IPRINTF ("DYAD UTIL: truncation might have happend with %s\n", - proclink); + IPRINTF ("DYAD UTIL: truncation might have happend with %s\n", proclink); } path[max_size + 1] = '\0'; @@ -446,7 +445,9 @@ int sync_containing_dir (const char* path) if (dir_fd < 0) { char errmsg[PATH_MAX + 256] = {'\0'}; - snprintf (errmsg, PATH_MAX + 256, "Failed to open directory %s\n", + snprintf (errmsg, + PATH_MAX + 256, + "Failed to open directory %s\n", containing_dir); perror (errmsg); return -1; // exit (SYS_ERR); @@ -454,16 +455,14 @@ int sync_containing_dir (const char* path) if (fsync (dir_fd) < 0) { char errmsg[PATH_MAX + 256] = {'\0'}; - snprintf (errmsg, PATH_MAX + 256, "Failed to fsync %s\n", - containing_dir); + snprintf (errmsg, PATH_MAX + 256, "Failed to fsync %s\n", containing_dir); perror (errmsg); return -1; // exit (SYS_ERR); } if (close (dir_fd) < 0) { char errmsg[PATH_MAX + 256] = {'\0'}; - snprintf (errmsg, PATH_MAX + 256, "Failed to close %s\n", - containing_dir); + snprintf (errmsg, PATH_MAX + 256, "Failed to close %s\n", containing_dir); perror (errmsg); return -1; // exit (SYS_ERR); } diff --git a/src/utils/utils.h b/src/utils/utils.h index 52173e82..734b17bd 100644 --- a/src/utils/utils.h +++ b/src/utils/utils.h @@ -18,7 +18,7 @@ #define DYAD_PATH_DELIM "/" #if defined(__cplusplus) -//#include // c++11 +// #include // c++11 #include #else #include diff --git a/src/wrapper/Makefile.am b/src/wrapper/Makefile.am index 94e212de..5cf9c674 100644 --- a/src/wrapper/Makefile.am +++ b/src/wrapper/Makefile.am @@ -1,17 +1,22 @@ lib_LTLIBRARIES = dyad_wrapper.la dyad_wrapper_la_SOURCES = wrapper.c dyad_wrapper_la_LDFLAGS = \ + -Wl,-rpath,'$(UCX_LIBDIR)' \ $(AM_LDFLAGS) \ -module \ -avoid-version \ -no-undefined \ - -shared \ - -export-symbols wrapper.sym + -shared dyad_wrapper_la_LIBADD = \ - $(top_builddir)/src/core/libdyad_core.la \ - # Need to add libutils.la now that libdyad_core is not exporting utils' symbols - $(top_builddir)/src/utils/libutils.la -dyad_wrapper_la_CPPFLAGS = -I$(top_builddir)/src/utils -I$(top_builddir)/src/core $(FLUX_CORE_CFLAGS) + $(top_builddir)/src/utils/libutils.la \ + $(top_builddir)/src/core/libdyad_core.la +dyad_wrapper_la_CPPFLAGS = \ + -I$(top_srcdir)/src/utils \ + -I$(top_srcdir)/src/core \ + -I$(top_srcdir)/src/dtl \ + $(FLUX_CORE_CFLAGS) \ + -DBUILDING_DYAD \ + -fvisibility=hidden install-exec-hook: @(cd $(DESTDIR)$(libdir) && $(RM) dyad_wrapper.la) diff --git a/src/wrapper/test/io_c.c b/src/wrapper/test/io_c.c index 1960921d..769e7c2c 100644 --- a/src/wrapper/test/io_c.c +++ b/src/wrapper/test/io_c.c @@ -45,8 +45,7 @@ int mkdir_of_path (const char* path) const mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); if (mkdir_as_needed (upper_dir, m) < 0) { char errmsg[PATH_MAX + 256] = {'\0'}; - snprintf (errmsg, PATH_MAX + 256, "Directory %s cannot be created.\n", - upper_dir); + snprintf (errmsg, PATH_MAX + 256, "Directory %s cannot be created.\n", upper_dir); perror (errmsg); return -1; } @@ -240,8 +239,11 @@ check_read (const char* buf, size_t sz, const char* line) size_t j = 0ul; for (; i < sz; ++i) { if (buffer[j] != buf[i]) { - fprintf (stderr, "error at %lu th character: %c != %c \n", i, - buffer[j], buf[i]); + fprintf (stderr, + "error at %lu th character: %c != %c \n", + i, + buffer[j], + buf[i]); return -2; } if (++j == buf_sz) diff --git a/src/wrapper/wrapper.c b/src/wrapper/wrapper.c index 8d7d2597..e771b32b 100644 --- a/src/wrapper/wrapper.c +++ b/src/wrapper/wrapper.c @@ -21,7 +21,7 @@ #include #include using namespace std; // std::clock () -//#include // c++11 +// #include // c++11 #else #include #include @@ -47,7 +47,7 @@ extern "C" { #endif static __thread dyad_ctx_t *ctx = NULL; -// static void dyad_wrapper_init (void) __attribute__((constructor)); +static void dyad_wrapper_init (void) __attribute__ ((constructor)); static void dyad_wrapper_fini (void) __attribute__ ((destructor)); #if DYAD_SYNC_DIR @@ -86,77 +86,20 @@ static inline int is_wronly (int fd) void dyad_wrapper_init (void) { - char *e = NULL; - - bool debug = false; - bool check = false; - bool shared_storage = false; - unsigned int key_depth = 0; - unsigned int key_bins = 0; - char *kvs_namespace = NULL; - char *prod_managed_path = NULL; - char *cons_managed_path = NULL; dyad_rc_t rc = DYAD_RC_OK; - if ((e = getenv (DYAD_SYNC_DEBUG_ENV))) { - debug = true; - enable_debug_dyad_utils (); - fprintf (stderr, "DYAD_WRAPPER: Initializeing DYAD wrapper\n"); - } else { - debug = false; - disable_debug_dyad_utils (); - } - - if ((e = getenv (DYAD_SYNC_CHECK_ENV))) - check = true; - else - check = false; - - if ((e = getenv (DYAD_SHARED_STORAGE_ENV)) && (atoi (e) != 0)) - shared_storage = true; - else - shared_storage = false; - - if ((e = getenv (DYAD_KEY_DEPTH_ENV))) - key_depth = atoi (e); - else - key_depth = 2; - - if ((e = getenv (DYAD_KEY_BINS_ENV))) - key_bins = atoi (e); - else - key_bins = 256; - - if ((e = getenv (DYAD_KVS_NAMESPACE_ENV))) - kvs_namespace = e; - else - kvs_namespace = NULL; - - if ((e = getenv (DYAD_PATH_CONSUMER_ENV))) { - cons_managed_path = e; - } else { - cons_managed_path = NULL; - } - if ((e = getenv (DYAD_PATH_PRODUCER_ENV))) { - prod_managed_path = e; - } else { - prod_managed_path = NULL; - } - - rc = dyad_init (debug, check, shared_storage, key_depth, key_bins, - kvs_namespace, prod_managed_path, cons_managed_path, &ctx); + rc = dyad_init_env (&ctx); if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERR (ctx, "Could not initialize DYAD!\n"); - ctx = NULL; + fprintf (stderr, "Failed to initialize DYAD (code = %d)\n", rc); + ctx->initialized = false; + ctx->reenter = false; return; } DYAD_LOG_INFO (ctx, "DYAD Initialized\n"); - DYAD_LOG_INFO (ctx, "%s=%s\n", DYAD_SYNC_DEBUG_ENV, - (ctx->debug) ? "true" : "false"); - DYAD_LOG_INFO (ctx, "%s=%s\n", DYAD_SYNC_CHECK_ENV, - (ctx->check) ? "true" : "false"); + DYAD_LOG_INFO (ctx, "%s=%s\n", DYAD_SYNC_DEBUG_ENV, (ctx->debug) ? "true" : "false"); + DYAD_LOG_INFO (ctx, "%s=%s\n", DYAD_SYNC_CHECK_ENV, (ctx->check) ? "true" : "false"); DYAD_LOG_INFO (ctx, "%s=%u\n", DYAD_KEY_DEPTH_ENV, ctx->key_depth); DYAD_LOG_INFO (ctx, "%s=%u\n", DYAD_KEY_BINS_ENV, ctx->key_bins); } @@ -169,17 +112,13 @@ void dyad_wrapper_fini () dyad_finalize (&ctx); } -int open (const char *path, int oflag, ...) +DYAD_DLL_EXPORTED int open (const char *path, int oflag, ...) { char *error = NULL; typedef int (*open_ptr_t) (const char *, int, mode_t, ...); open_ptr_t func_ptr = NULL; int mode = 0; - if (ctx == NULL) { - dyad_wrapper_init (); - } - if (oflag & O_CREAT) { va_list arg; va_start (arg, oflag); @@ -199,8 +138,7 @@ int open (const char *path, int oflag, ...) } if (!(ctx && ctx->h) || (ctx && !ctx->reenter)) { - IPRINTF (ctx, "DYAD_SYNC: open sync not applicable for \"%s\".\n", - path); + IPRINTF (ctx, "DYAD_SYNC: open sync not applicable for \"%s\".\n", path); goto real_call; } @@ -216,16 +154,12 @@ real_call:; return (func_ptr (path, oflag, mode)); } -FILE *fopen (const char *path, const char *mode) +DYAD_DLL_EXPORTED FILE *fopen (const char *path, const char *mode) { char *error = NULL; typedef FILE *(*fopen_ptr_t) (const char *, const char *); fopen_ptr_t func_ptr = NULL; - if (ctx == NULL) { - dyad_wrapper_init (); - } - func_ptr = (fopen_ptr_t)dlsym (RTLD_NEXT, "fopen"); if ((error = dlerror ())) { DPRINTF (ctx, "DYAD_SYNC: error in dlsym: %s\n", error); @@ -238,7 +172,8 @@ FILE *fopen (const char *path, const char *mode) } if (!(ctx && ctx->h) || (ctx && !ctx->reenter) || !path) { - IPRINTF (ctx, "DYAD_SYNC: fopen sync not applicable for \"%s\".\n", + IPRINTF (ctx, + "DYAD_SYNC: fopen sync not applicable for \"%s\".\n", ((path) ? path : "")); goto real_call; } @@ -254,7 +189,7 @@ FILE *fopen (const char *path, const char *mode) return (func_ptr (path, mode)); } -int close (int fd) +DYAD_DLL_EXPORTED int close (int fd) { bool to_sync = false; char *error = NULL; @@ -263,10 +198,6 @@ int close (int fd) char path[PATH_MAX + 1] = {'\0'}; int rc = 0; - if (ctx == NULL) { - dyad_wrapper_init (); - } - func_ptr = (close_ptr_t)dlsym (RTLD_NEXT, "close"); if ((error = dlerror ())) { DPRINTF (ctx, "DYAD_SYNC: error in dlsym: %s\n", error); @@ -276,13 +207,11 @@ int close (int fd) if ((fd < 0) || (ctx == NULL) || (ctx->h == NULL) || !ctx->reenter) { #if defined(IPRINTF_DEFINED) if (ctx == NULL) { - IPRINTF (ctx, - "DYAD_SYNC: close sync not applicable. (no context)\n"); + IPRINTF (ctx, "DYAD_SYNC: close sync not applicable. (no context)\n"); } else if (ctx->h == NULL) { IPRINTF (ctx, "DYAD_SYNC: close sync not applicable. (no flux)\n"); } else if (!ctx->reenter) { - IPRINTF (ctx, - "DYAD_SYNC: close sync not applicable. (no reenter)\n"); + IPRINTF (ctx, "DYAD_SYNC: close sync not applicable. (no reenter)\n"); } else if (fd >= 0) { IPRINTF (ctx, "DYAD_SYNC: close sync not applicable. (invalid file " @@ -299,8 +228,7 @@ int close (int fd) } if (get_path (fd, PATH_MAX - 1, path) < 0) { - IPRINTF (ctx, - "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); + IPRINTF (ctx, "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); to_sync = false; goto real_call; } @@ -321,15 +249,15 @@ real_call:; // semicolon here to avoid the error int wronly = is_wronly (fd); if (wronly == -1) { - DPRINTF (ctx, "Failed to check the mode of the file with fcntl: %s\n", + DPRINTF (ctx, + "Failed to check the mode of the file with fcntl: %s\n", strerror (errno)); } if (to_sync && wronly == 1) { rc = func_ptr (fd); if (rc != 0) { - DPRINTF (ctx, "Failed close (\"%s\").: %s\n", path, - strerror (errno)); + DPRINTF (ctx, "Failed close (\"%s\").: %s\n", path, strerror (errno)); } IPRINTF (ctx, "DYAD_SYNC: enters close sync (\"%s\").\n", path); if (DYAD_IS_ERROR (dyad_produce (ctx, path))) { @@ -343,7 +271,7 @@ real_call:; // semicolon here to avoid the error return rc; } -int fclose (FILE *fp) +DYAD_DLL_EXPORTED int fclose (FILE *fp) { bool to_sync = false; char *error = NULL; @@ -353,10 +281,6 @@ int fclose (FILE *fp) int rc = 0; int fd = 0; - if (ctx == NULL) { - dyad_wrapper_init (); - } - func_ptr = (fclose_ptr_t)dlsym (RTLD_NEXT, "fclose"); if ((error = dlerror ())) { DPRINTF (ctx, "DYAD_SYNC: error in dlsym: %s\n", error); @@ -366,13 +290,11 @@ int fclose (FILE *fp) if ((fp == NULL) || (ctx == NULL) || (ctx->h == NULL) || !ctx->reenter) { #if defined(IPRINTF_DEFINED) if (ctx == NULL) { - IPRINTF (ctx, - "DYAD_SYNC: fclose sync not applicable. (no context)\n"); + IPRINTF (ctx, "DYAD_SYNC: fclose sync not applicable. (no context)\n"); } else if (ctx->h == NULL) { IPRINTF (ctx, "DYAD_SYNC: fclose sync not applicable. (no flux)\n"); } else if (!ctx->reenter) { - IPRINTF (ctx, - "DYAD_SYNC: fclose sync not applicable. (no reenter)\n"); + IPRINTF (ctx, "DYAD_SYNC: fclose sync not applicable. (no reenter)\n"); } else if (fp == NULL) { IPRINTF (ctx, "DYAD_SYNC: fclose sync not applicable. (invalid file " @@ -389,8 +311,7 @@ int fclose (FILE *fp) } if (get_path (fileno (fp), PATH_MAX - 1, path) < 0) { - IPRINTF (ctx, - "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); + IPRINTF (ctx, "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); to_sync = false; goto real_call; } @@ -411,7 +332,8 @@ real_call:; int wronly = is_wronly (fd); if (wronly == -1) { - DPRINTF (ctx, "Failed to check the mode of the file with fcntl: %s\n", + DPRINTF (ctx, + "Failed to check the mode of the file with fcntl: %s\n", strerror (errno)); } diff --git a/src/wrapper/wrapper.sym b/src/wrapper/wrapper.sym deleted file mode 100644 index 87d3bc26..00000000 --- a/src/wrapper/wrapper.sym +++ /dev/null @@ -1,4 +0,0 @@ -open -fopen -close -fclose