Skip to content

Commit 53e9a95

Browse files
StellarBotJiakunYan
StellarBot
and
JiakunYan
committed
Merge #6363
6363: Update the lci parcelport to use LCI v1.7.6 r=hkaiser a=JiakunYan The major changes include: - Necessary changes to CMake files to accommodate changes due to the split of the original LCI library into two libraries (LCI and LCT). - A new performance counters and logging infrastructure based on LCT (currently only applied to the lci pp). - Controlled by CMake variables `HPX_WITH_PARCELPORT_LCI_PCOUNTER` and `HPX_WITH_PARCELPORT_LCI_LOG`. The default is `OFF`. - New LCI parcelport configurations: - `reg_mem`: whether to explicitly register memory buffers for long messages (value 1) or just let LCI register them on the fly (value 0). The default is 1. - `ndevices`: how many LCI devices (low-level network contexts) to use. The default is 2. - `ncomps`: how many completion managers to use. The default is 1. Co-authored-by: Jiakun Yan <[email protected]>
2 parents a0c6d49 + 67dd637 commit 53e9a95

22 files changed

+695
-277
lines changed

‎CMakeLists.txt

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1147,10 +1147,28 @@ if(HPX_WITH_NETWORKING)
11471147
ADVANCED
11481148
)
11491149
hpx_option(
1150-
HPX_WITH_LCI_TAG STRING "LCI repository tag or branch" "v1.7.5"
1150+
HPX_WITH_LCI_TAG STRING "LCI repository tag or branch" "v1.7.6"
11511151
CATEGORY "Build Targets"
11521152
ADVANCED
11531153
)
1154+
hpx_option(
1155+
HPX_WITH_PARCELPORT_LCI_LOG STRING
1156+
"Enable the LCI-parcelport-specific logger" OFF
1157+
CATEGORY "Parcelport"
1158+
ADVANCED
1159+
)
1160+
if(HPX_WITH_PARCELPORT_LCI_LOG)
1161+
hpx_add_config_define(HPX_HAVE_PARCELPORT_LCI_LOG)
1162+
endif()
1163+
hpx_option(
1164+
HPX_WITH_PARCELPORT_LCI_PCOUNTER STRING
1165+
"Enable the LCI-parcelport-specific performance counter" OFF
1166+
CATEGORY "Parcelport"
1167+
ADVANCED
1168+
)
1169+
if(HPX_WITH_PARCELPORT_LCI_PCOUNTER)
1170+
hpx_add_config_define(HPX_HAVE_PARCELPORT_LCI_PCOUNTER)
1171+
endif()
11541172

11551173
hpx_option(
11561174
HPX_WITH_PARCELPORT_GASNET BOOL "Enable the GASNET based parcelport." OFF

‎cmake/HPX_SetupLCI.cmake

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,23 +87,24 @@ macro(hpx_setup_lci)
8787
endif()
8888

8989
install(
90-
TARGETS LCI lci-ucx
90+
TARGETS LCI LCT lci-ucx
9191
EXPORT HPXLCITarget
9292
COMPONENT core
9393
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
9494
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
9595
)
9696

9797
install(
98-
DIRECTORY ${lci_SOURCE_DIR}/src/api/ ${lci_BINARY_DIR}/src/api/
98+
DIRECTORY ${lci_SOURCE_DIR}/lci/api/ ${lci_BINARY_DIR}/lci/api/
99+
${lci_SOURCE_DIR}/lct/api/ ${lci_BINARY_DIR}/lct/api/
99100
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
100101
COMPONENT core
101102
FILES_MATCHING
102103
PATTERN "*.h"
103104
)
104105

105106
export(
106-
TARGETS LCI lci-ucx
107+
TARGETS LCI LCT lci-ucx
107108
NAMESPACE LCI::
108109
FILE "${CMAKE_CURRENT_BINARY_DIR}/lib/cmake/${HPX_PACKAGE_NAME}/HPXLCITarget.cmake"
109110
)

‎libs/core/lci_base/include/hpx/lci_base/lci_environment.hpp

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,22 +42,55 @@ namespace hpx { namespace util {
4242

4343
static std::string get_processor_name();
4444

45-
// Configurations:
46-
// Log level
45+
// log
4746
enum class log_level_t
4847
{
4948
none,
5049
profile,
51-
debug
50+
debug,
5251
};
5352
static log_level_t log_level;
54-
// Output filename of log
55-
static FILE* log_outfile;
56-
static void log(log_level_t level, const char* format, ...);
57-
58-
private:
59-
static bool enabled_;
60-
};
53+
#ifdef HPX_HAVE_PARCELPORT_LCI_LOG
54+
static LCT_log_ctx_t log_ctx;
55+
#endif
56+
static void log(
57+
log_level_t level, const char* tag, const char* format, ...);
58+
// performance counter
59+
// clang-format off
60+
#define HPX_LCI_PCOUNTER_NONE_FOR_EACH(_macro)
61+
62+
#define HPX_LCI_PCOUNTER_TREND_FOR_EACH(_macro) \
63+
_macro(send_conn_start) \
64+
_macro(send_conn_end) \
65+
_macro(recv_conn_start) \
66+
_macro(recv_conn_end)
67+
68+
#define HPX_LCI_PCOUNTER_TIMER_FOR_EACH(_macro) \
69+
_macro(send_conn_timer) \
70+
_macro(recv_conn_timer) \
71+
_macro(async_write_timer) \
72+
_macro(send_timer) \
73+
_macro(handle_parcels) \
74+
_macro(poll_comp) \
75+
_macro(useful_bg_work)
76+
// clang-format on
77+
78+
#define HPX_LCI_PCOUNTER_HANDLE_DECL(name) static LCT_pcounter_handle_t name;
79+
80+
HPX_LCI_PCOUNTER_NONE_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DECL)
81+
HPX_LCI_PCOUNTER_TREND_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DECL)
82+
HPX_LCI_PCOUNTER_TIMER_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DECL)
83+
84+
static LCT_pcounter_ctx_t pcounter_ctx;
85+
static int64_t pcounter_now();
86+
static int64_t pcounter_since(int64_t then);
87+
static void pcounter_add(LCT_pcounter_handle_t handle, int64_t val);
88+
static void pcounter_start(LCT_pcounter_handle_t handle);
89+
static void pcounter_end(LCT_pcounter_handle_t handle);
90+
91+
private:
92+
static bool enabled_;
93+
};
6194
}} // namespace hpx::util
6295

6396
#include <hpx/config/warnings_suffix.hpp>

‎libs/core/lci_base/src/lci_environment.cpp

Lines changed: 95 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -91,60 +91,19 @@ namespace hpx { namespace util {
9191
defined(HPX_HAVE_MODULE_LCI_BASE)
9292

9393
namespace hpx { namespace util {
94-
9594
bool lci_environment::enabled_ = false;
96-
lci_environment::log_level_t lci_environment::log_level = log_level_t::none;
97-
FILE* lci_environment::log_outfile = nullptr;
95+
lci_environment::log_level_t lci_environment::log_level;
96+
#ifdef HPX_HAVE_PARCELPORT_LCI_LOG
97+
LCT_log_ctx_t lci_environment::log_ctx;
98+
#endif
99+
LCT_pcounter_ctx_t lci_environment::pcounter_ctx;
98100

99-
///////////////////////////////////////////////////////////////////////////
100-
void lci_environment::init_config(util::runtime_configuration& rtcfg)
101-
{
102-
// The default value here does not matter here
103-
std::string log_level_str = get_entry_as<std::string>(
104-
rtcfg, "hpx.parcel.lci.log_level", "" /* Does not matter*/);
105-
if (log_level_str == "none")
106-
log_level = log_level_t::none;
107-
else if (log_level_str == "profile")
108-
log_level = log_level_t::profile;
109-
else if (log_level_str == "debug")
110-
log_level = log_level_t::debug;
111-
else
112-
throw std::runtime_error("Unknown log level " + log_level_str);
113-
std::string log_filename = get_entry_as<std::string>(
114-
rtcfg, "hpx.parcel.lci.log_outfile", "" /* Does not matter*/);
115-
if (log_filename == "stderr")
116-
log_outfile = stderr;
117-
else if (log_filename == "stdout")
118-
log_outfile = stdout;
119-
else
120-
{
121-
const int filename_max = 256;
122-
char filename[filename_max];
123-
char* p0_old = log_filename.data();
124-
char* p0_new = strchr(log_filename.data(), '%');
125-
char* p1 = filename;
126-
while (p0_new)
127-
{
128-
long nbytes = p0_new - p0_old;
129-
HPX_ASSERT(p1 + nbytes < filename + filename_max);
130-
memcpy(p1, p0_old, nbytes);
131-
p1 += nbytes;
132-
nbytes =
133-
snprintf(p1, filename + filename_max - p1, "%d", LCI_RANK);
134-
p1 += nbytes;
135-
p0_old = p0_new + 1;
136-
p0_new = strchr(p0_old, '%');
137-
}
138-
strncat(p1, p0_old, filename + filename_max - p1 - 1);
139-
log_outfile = fopen(filename, "w+");
140-
if (log_outfile == nullptr)
141-
{
142-
throw std::runtime_error(
143-
"Cannot open the logfile " + std::string(filename));
144-
}
145-
}
146-
}
101+
#define HPX_LCI_PCOUNTER_HANDLE_DEF(name) \
102+
LCT_pcounter_handle_t lci_environment::name;
147103

104+
HPX_LCI_PCOUNTER_NONE_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DEF)
105+
HPX_LCI_PCOUNTER_TREND_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DEF)
106+
HPX_LCI_PCOUNTER_TIMER_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DEF)
148107
///////////////////////////////////////////////////////////////////////////
149108
void lci_environment::init(
150109
int*, char***, util::runtime_configuration& rtcfg)
@@ -186,7 +145,34 @@ namespace hpx { namespace util {
186145

187146
rtcfg.add_entry("hpx.parcel.bootstrap", "lci");
188147
rtcfg.add_entry("hpx.parcel.lci.rank", std::to_string(this_rank));
189-
init_config(rtcfg);
148+
LCT_init();
149+
// initialize the log context
150+
#ifdef HPX_HAVE_PARCELPORT_LCI_LOG
151+
const char* const log_levels[] = {"none", "profile", "debug"};
152+
log_ctx = LCT_log_ctx_alloc(log_levels,
153+
sizeof(log_levels) / sizeof(log_levels[0]), 0, "hpx_lci",
154+
getenv("HPX_LCI_LOG_OUTFILE"), getenv("HPX_LCI_LOG_LEVEL"),
155+
getenv("HPX_LCI_LOG_WHITELIST"), getenv("HPX_LCI_LOG_BLACKLIST"));
156+
log_level = static_cast<log_level_t>(LCT_log_get_level(log_ctx));
157+
#else
158+
log_level = log_level_t::none;
159+
#endif
160+
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
161+
// initialize the performance counters
162+
pcounter_ctx = LCT_pcounter_ctx_alloc("hpx-lci");
163+
164+
#define HPX_LCI_PCOUNTER_NONE_REGISTER(name) \
165+
name = LCT_pcounter_register(pcounter_ctx, #name, LCT_PCOUNTER_NONE);
166+
HPX_LCI_PCOUNTER_NONE_FOR_EACH(HPX_LCI_PCOUNTER_NONE_REGISTER)
167+
168+
#define HPX_LCI_PCOUNTER_TREND_REGISTER(name) \
169+
name = LCT_pcounter_register(pcounter_ctx, #name, LCT_PCOUNTER_TREND);
170+
HPX_LCI_PCOUNTER_TREND_FOR_EACH(HPX_LCI_PCOUNTER_TREND_REGISTER)
171+
172+
#define HPX_LCI_PCOUNTER_TIMER_REGISTER(name) \
173+
name = LCT_pcounter_register(pcounter_ctx, #name, LCT_PCOUNTER_TIMER);
174+
HPX_LCI_PCOUNTER_TIMER_FOR_EACH(HPX_LCI_PCOUNTER_TIMER_REGISTER)
175+
#endif
190176
enabled_ = true;
191177
}
192178

@@ -200,7 +186,13 @@ namespace hpx { namespace util {
200186
if (enabled())
201187
{
202188
enabled_ = false;
203-
// for some reasons, this code block can be entered twice when HPX exits
189+
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
190+
LCT_pcounter_ctx_free(&pcounter_ctx);
191+
#endif
192+
#ifdef HPX_HAVE_PARCELPORT_LCI_LOG
193+
LCT_log_ctx_free(&log_ctx);
194+
#endif
195+
LCT_fina();
204196
int lci_init = 0;
205197
LCI_initialized(&lci_init);
206198
if (lci_init)
@@ -240,17 +232,63 @@ namespace hpx { namespace util {
240232
return res;
241233
}
242234

243-
void lci_environment::log(
244-
lci_environment::log_level_t level, const char* format, ...)
235+
void lci_environment::log([[maybe_unused]] log_level_t level,
236+
[[maybe_unused]] const char* tag, [[maybe_unused]] const char* format,
237+
...)
245238
{
239+
#ifdef HPX_HAVE_PARCELPORT_LCI_LOG
240+
if (level > log_level)
241+
return;
246242
va_list args;
247243
va_start(args, format);
248244

249-
if (level <= log_level)
250-
vfprintf(log_outfile, format, args);
245+
LCT_Logv(log_ctx, static_cast<int>(level), tag, format, args);
251246

252247
va_end(args);
248+
#endif
249+
}
250+
251+
int64_t lci_environment::pcounter_now()
252+
{
253+
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
254+
return static_cast<int64_t>(LCT_now());
255+
#endif
256+
return 0;
257+
}
258+
259+
int64_t lci_environment::pcounter_since([[maybe_unused]] int64_t then)
260+
{
261+
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
262+
return static_cast<int64_t>(LCT_now()) - then;
263+
#endif
264+
return 0;
265+
}
266+
267+
void lci_environment::pcounter_add(
268+
[[maybe_unused]] LCT_pcounter_handle_t handle,
269+
[[maybe_unused]] int64_t val)
270+
{
271+
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
272+
LCT_pcounter_add(pcounter_ctx, handle, val);
273+
#endif
274+
}
275+
276+
void lci_environment::pcounter_start(
277+
[[maybe_unused]] LCT_pcounter_handle_t handle)
278+
{
279+
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
280+
LCT_pcounter_start(pcounter_ctx, handle);
281+
#endif
253282
}
283+
284+
void lci_environment::pcounter_end(
285+
[[maybe_unused]] LCT_pcounter_handle_t handle)
286+
{
287+
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
288+
LCT_pcounter_end(pcounter_ctx, handle);
289+
#endif
290+
}
291+
254292
}} // namespace hpx::util
255293

256294
#endif

‎libs/full/parcelport_lci/include/hpx/parcelport_lci/completion_manager/completion_manager_queue.hpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@ namespace hpx::parcelset::policies::lci {
1717
{
1818
completion_manager_queue()
1919
{
20-
LCI_queue_create(LCI_UR_DEVICE, &queue);
20+
// LCI_queue_create(LCI_UR_DEVICE, &queue);
21+
// Hack for now
22+
LCI_queue_createx(LCI_UR_DEVICE,
23+
LCI_SERVER_NUM_PKTS * (size_t) config_t::ndevices, &queue);
2124
}
2225

2326
~completion_manager_queue()

‎libs/full/parcelport_lci/include/hpx/parcelport_lci/config.hpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@ namespace hpx::parcelset::policies::lci {
4545
// How many pre-posted receives for new messages
4646
// (can only be applied to `sendrecv` protocol).
4747
static int prepost_recv_num;
48+
// Whether to register the buffer in HPX (or rely on LCI to register it)
49+
static bool reg_mem;
50+
// How many devices to use
51+
static int ndevices;
52+
// How many completion managers to use
53+
static int ncomps;
4854

4955
static void init_config(util::runtime_configuration const& rtcfg);
5056
};

‎libs/full/parcelport_lci/include/hpx/parcelport_lci/header.hpp

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,23 +28,25 @@ namespace hpx::parcelset::policies::lci {
2828
{
2929
// siguature for assert_valid
3030
pos_signature = 0,
31+
// device idx
32+
pos_device_idx = 1 * sizeof(value_type),
3133
// tag
32-
pos_tag = 1 * sizeof(value_type),
34+
pos_tag = 2 * sizeof(value_type),
3335
// non-zero-copy chunk size
34-
pos_numbytes_nonzero_copy = 2 * sizeof(value_type),
36+
pos_numbytes_nonzero_copy = 3 * sizeof(value_type),
3537
// transmission chunk size
36-
pos_numbytes_tchunk = 3 * sizeof(value_type),
38+
pos_numbytes_tchunk = 4 * sizeof(value_type),
3739
// how many bytes in total (including zero-copy and non-zero-copy chunks)
38-
pos_numbytes = 4 * sizeof(value_type),
40+
pos_numbytes = 5 * sizeof(value_type),
3941
// zero-copy chunk number
40-
pos_numchunks_zero_copy = 5 * sizeof(value_type),
42+
pos_numchunks_zero_copy = 6 * sizeof(value_type),
4143
// non-zero-copy chunk number
42-
pos_numchunks_nonzero_copy = 6 * sizeof(value_type),
44+
pos_numchunks_nonzero_copy = 7 * sizeof(value_type),
4345
// whether piggyback data
44-
pos_piggy_back_flag_data = 7 * sizeof(value_type),
46+
pos_piggy_back_flag_data = 8 * sizeof(value_type),
4547
// whether piggyback transmission chunk
46-
pos_piggy_back_flag_tchunk = 7 * sizeof(value_type) + 1,
47-
pos_piggy_back_address = 7 * sizeof(value_type) + 2
48+
pos_piggy_back_flag_tchunk = 8 * sizeof(value_type) + 1,
49+
pos_piggy_back_address = 8 * sizeof(value_type) + 2
4850
};
4951

5052
template <typename buffer_type, typename ChunkType>
@@ -133,6 +135,16 @@ namespace hpx::parcelset::policies::lci {
133135
return get<pos_signature>();
134136
}
135137

138+
void set_device_idx(int device_idx) noexcept
139+
{
140+
set<pos_device_idx>(static_cast<value_type>(device_idx));
141+
}
142+
143+
value_type get_device_idx() const noexcept
144+
{
145+
return get<pos_device_idx>();
146+
}
147+
136148
void set_tag(LCI_tag_t tag) noexcept
137149
{
138150
set<pos_tag>(static_cast<value_type>(tag));

0 commit comments

Comments
 (0)