Skip to content

Commit 3c9d69a

Browse files
hariharan-devarajanhariharan-devarajan
committed
Merge branch 'feature/unittests' of github.com:flux-framework/dyad into feature/unittests
2 parents 2f163ce + 93d6516 commit 3c9d69a

File tree

10 files changed

+614
-83
lines changed

10 files changed

+614
-83
lines changed

‎tests/dspaces_perf/data_plane/data_plane.cpp

Lines changed: 124 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55

66
#include <array>
77

8+
#define NS_TO_SECS(ns_var) ((double) ns_var / 1000000000.0)
9+
#define AGG_TIME(time_var, agg_time_var, dtype) MPI_Reduce(&time_var, &agg_time_var, 1, dtype, MPI_SUM, 0, MPI_COMM_WORLD)
10+
811
FILE* redirect_stdout(const char* filename)
912
{
1013
size_t dir_len = strlen(args.dspaces_timing_dir.c_str());
@@ -87,179 +90,224 @@ int create_files_per_server_process(dspaces_client_t* client, bool is_local, boo
8790
return rc;
8891
}
8992

93+
void gen_perf_print(size_t data_len, double total_mdata_time, double total_data_time)
94+
{
95+
double agg_mdata_time = 0;
96+
double agg_data_time = 0;
97+
AGG_TIME(total_mdata_time, agg_mdata_time, MPI_DOUBLE);
98+
AGG_TIME(total_data_time, agg_data_time, MPI_DOUBLE);
99+
if (info.rank == 0) {
100+
double final_mdata_time = agg_mdata_time / info.comm_size;
101+
double final_data_time = agg_data_time / info.comm_size;
102+
printf("[DSPACES_TEST],%10d,%10lu,%10lu,%10.6f,%10.6f,%10.6f,%10.6f\n",
103+
info.comm_size, // Comm Size
104+
data_len*args.number_of_files, // Total I/O per process
105+
args.number_of_files, // Number of mdata ops per process
106+
final_mdata_time, // Metadata Time
107+
final_data_time, // Data Time
108+
data_len*args.number_of_files*info.comm_size/final_mdata_time/1024/1024.0, // Metadata Bandwidth
109+
args.number_of_files*info.comm_size/final_data_time/1024/1024.0 // Data Bandwidth
110+
);
111+
}
112+
}
113+
90114
TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]"
91115
"[file_size= " + std::to_string(args.request_size*args.iteration) +"]"
92116
"[parallel_req= " + std::to_string(info.comm_size) +"]"
93117
"[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") {
94-
REQUIRE (pretest() == 0);
95-
dspaces_client_t client = dspaces_CLIENT_NULL;
96-
int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client);
97-
REQUIRE (rc == dspaces_SUCCESS);
98-
REQUIRE (create_files_per_server_process(&client, false, true) == 0);
99118
SECTION("Test Max Bandwidth") {
100119
Timer data_time;
120+
REQUIRE (pretest() == 0);
121+
dspaces_client_t client = dspaces_CLIENT_NULL;
122+
int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client);
123+
REQUIRE (rc == dspaces_SUCCESS);
124+
REQUIRE (create_files_per_server_process(&client, false, true) == 0);
101125
char filename[4096];
102126
gen_var_name(filename, false, true, false, true);
103127
size_t data_len = args.request_size*args.iteration;
104128
char* file_data = NULL;
105129
int ndim = 1;
106130
uint64_t lb = 0;
107131
uint64_t ub = data_len - 1;
108-
FILE* fp = redirect_stdout("remote_data_bandwidth.csv");
109-
REQUIRE(fp != NULL);
110-
printf("rank,var_name,version,mdata_time_ns,data_time_ns\n");
132+
char csv_filename[4096];
133+
// sprintf(csv_filename, "remote_data_bandwidth_%d.csv", info.rank);
134+
// FILE* fp = fopen(csv_filename, "w+");
135+
// FILE* fp = redirect_stdout(csv_filename);
136+
// REQUIRE(fp != NULL);
137+
// printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n");
138+
double total_mdata_time = 0;
139+
double total_data_time = 0;
111140
for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) {
141+
long long int mdata_time_ns = 0;
142+
long long int data_time_ns = 0;
112143
data_time.resumeTime();
113144
// Using aget instead of get because dyad_get_data also allocates the buffer
114145
// Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available.
115-
rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1);
146+
rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns);
116147
data_time.pauseTime();
117148
REQUIRE (rc == dspaces_SUCCESS);
118149
free(file_data);
150+
total_mdata_time += NS_TO_SECS(mdata_time_ns);
151+
total_data_time += NS_TO_SECS(data_time_ns);
119152
}
120-
restore_stdout(fp);
121-
AGGREGATE_TIME(data);
122-
if (info.rank == 0) {
123-
printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n",
124-
info.comm_size, data_len*args.number_of_files,
125-
total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0);
126-
}
153+
// restore_stdout(fp);
154+
// AGGREGATE_TIME(data);
155+
gen_perf_print(data_len, total_mdata_time, total_data_time);
156+
rc = dspaces_fini(client);
157+
REQUIRE (rc == dspaces_SUCCESS);
158+
// fclose(fp);
159+
REQUIRE (posttest() == 0);
127160
}
128-
rc = dspaces_fini(client);
129-
REQUIRE (rc == dspaces_SUCCESS);
130-
REQUIRE (posttest() == 0);
131161
}
132162

133163
TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_files) +"]"
134164
"[file_size= " + std::to_string(args.request_size*args.iteration) +"]"
135165
"[parallel_req= " + std::to_string(info.comm_size) +"]"
136166
"[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") {
137-
REQUIRE (pretest() == 0);
138-
dspaces_client_t client = dspaces_CLIENT_NULL;
139-
int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client);
140-
REQUIRE (rc == dspaces_SUCCESS);
141-
REQUIRE (create_files_per_server_process(&client, false, false) == 0);
142167
SECTION("Test Max Bandwidth") {
143168
Timer data_time;
169+
REQUIRE (pretest() == 0);
170+
dspaces_client_t client = dspaces_CLIENT_NULL;
171+
int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client);
172+
REQUIRE (rc == dspaces_SUCCESS);
173+
REQUIRE (create_files_per_server_process(&client, false, false) == 0);
144174
char filename[4096];
145175
gen_var_name(filename, false, false, false, true);
146176
char* file_data = NULL;
147177
size_t data_len = args.request_size*args.iteration;
148178
int ndim = 1;
149179
uint64_t lb = 0;
150180
uint64_t ub = data_len - 1;
151-
FILE* fp = redirect_stdout("remote_data_agg_bandwidth.csv");
152-
REQUIRE(fp != NULL);
153-
printf("rank,var_name,version,mdata_time_ns,data_time_ns\n");
181+
char csv_filename[4096];
182+
// sprintf(csv_filename, "remote_data_agg_bandwidth_%d.csv", info.rank);
183+
// FILE* fp = redirect_stdout(csv_filename);
184+
// FILE* fp = fopen(csv_filename, "w+");
185+
// REQUIRE(fp != NULL);
186+
// printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n");
187+
double total_mdata_time = 0;
188+
double total_data_time = 0;
154189
if (info.rank % args.process_per_node != 0)
155190
usleep (10000);
156191
for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) {
192+
long long int mdata_time_ns = 0;
193+
long long int data_time_ns = 0;
157194
data_time.resumeTime();
158195
// Using aget instead of get because dyad_get_data also allocates the buffer
159196
// Unlike the previous test, we set the timeout to -1 so it will do any blocking that it might want to do
160197
// TODO: confirm that the timeout is actually needed to guarantee this type of behavior
161-
rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1);
198+
rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns);
162199
data_time.pauseTime();
163200
REQUIRE (rc == dspaces_SUCCESS);
164201
free(file_data);
202+
total_mdata_time += NS_TO_SECS(mdata_time_ns);
203+
total_data_time += NS_TO_SECS(data_time_ns);
165204
}
166-
restore_stdout(fp);
167-
AGGREGATE_TIME(data);
168-
if (info.rank == 0) {
169-
printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n",
170-
info.comm_size, data_len*args.number_of_files,
171-
total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0);
172-
}
205+
// restore_stdout(fp);
206+
// AGGREGATE_TIME(data);
207+
gen_perf_print(data_len, total_mdata_time, total_data_time);
208+
rc = dspaces_fini(client);
209+
REQUIRE (rc == dspaces_SUCCESS);
210+
// fclose(fp);
211+
REQUIRE (posttest() == 0);
173212
}
174-
rc = dspaces_fini(client);
175-
REQUIRE (rc == dspaces_SUCCESS);
176-
REQUIRE (posttest() == 0);
177213
}
178214

179215

180216
TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]"
181217
"[file_size= " + std::to_string(args.request_size*args.iteration) +"]"
182218
"[parallel_req= " + std::to_string(info.comm_size) +"]"
183219
"[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") {
184-
REQUIRE (pretest() == 0);
185-
dspaces_client_t client = dspaces_CLIENT_NULL;
186-
int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client);
187-
REQUIRE (rc == dspaces_SUCCESS);
188-
REQUIRE (create_files_per_server_process(&client, true, true) == 0);
189220
SECTION("Test Max Bandwidth") {
190221
Timer data_time;
222+
REQUIRE (pretest() == 0);
223+
dspaces_client_t client = dspaces_CLIENT_NULL;
224+
int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client);
225+
REQUIRE (rc == dspaces_SUCCESS);
226+
REQUIRE (create_files_per_server_process(&client, true, true) == 0);
191227
char filename[4096];
192228
gen_var_name(filename, true, false, false, false);
193229
size_t data_len = args.request_size*args.iteration;
194230
int ndim = 1;
195231
uint64_t lb = 0;
196232
uint64_t ub = data_len - 1;
197233
char* file_data = NULL;
198-
FILE* fp = redirect_stdout("local_process_data_bandwidth.csv");
199-
REQUIRE(fp != NULL);
200-
printf("rank,var_name,version,mdata_time_ns,data_time_ns\n");
234+
char csv_filename[4096];
235+
// sprintf(csv_filename, "local_process_data_bandwidth_%d.csv", info.rank);
236+
// FILE* fp = redirect_stdout(csv_filename);
237+
// FILE* fp = fopen(csv_filename, "w+");
238+
// REQUIRE(fp != NULL);
239+
// printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n");
240+
double total_mdata_time = 0;
241+
double total_data_time = 0;
201242
for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) {
243+
long long int mdata_time_ns = 0;
244+
long long int data_time_ns = 0;
202245
data_time.resumeTime();
203246
// Using aget instead of get because dyad_get_data also allocates the buffer
204247
// Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available.
205-
rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1);
248+
rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns);
206249
data_time.pauseTime();
207250
REQUIRE (rc == dspaces_SUCCESS);
208251
free(file_data);
252+
total_mdata_time += NS_TO_SECS(mdata_time_ns);
253+
total_data_time += NS_TO_SECS(data_time_ns);
209254
}
210-
restore_stdout(fp);
211-
AGGREGATE_TIME(data);
212-
if (info.rank == 0) {
213-
printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n",
214-
info.comm_size, data_len*args.number_of_files,
215-
total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0);
216-
}
255+
// restore_stdout(fp);
256+
// AGGREGATE_TIME(data);
257+
gen_perf_print(data_len, total_mdata_time, total_data_time);
258+
rc = dspaces_fini(client);
259+
REQUIRE (rc == dspaces_SUCCESS);
260+
// fclose(fp);
261+
REQUIRE (posttest() == 0);
217262
}
218-
rc = dspaces_fini(client);
219-
REQUIRE (rc == dspaces_SUCCESS);
220-
REQUIRE (posttest() == 0);
221263
}
222264

223265

224266
TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]"
225267
"[file_size= " + std::to_string(args.request_size*args.iteration) +"]"
226268
"[parallel_req= " + std::to_string(info.comm_size) +"]"
227269
"[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") {
228-
REQUIRE (pretest() == 0);
229-
dspaces_client_t client = dspaces_CLIENT_NULL;
230-
int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client);
231-
REQUIRE (rc == dspaces_SUCCESS);
232-
REQUIRE (create_files_per_server_process(&client, true, true) == 0);
233270
SECTION("Test Max Bandwidth") {
234271
Timer data_time;
272+
REQUIRE (pretest() == 0);
273+
dspaces_client_t client = dspaces_CLIENT_NULL;
274+
int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client);
275+
REQUIRE (rc == dspaces_SUCCESS);
276+
REQUIRE (create_files_per_server_process(&client, true, true) == 0);
235277
char filename[4096];
236278
gen_var_name(filename, true, false, true, false);
237279
size_t data_len = args.request_size*args.iteration;
238280
int ndim = 1;
239281
uint64_t lb = 0;
240282
uint64_t ub = data_len - 1;
241283
char* file_data = NULL;
242-
FILE* fp = redirect_stdout("local_node_data_bandwidth.csv");
243-
REQUIRE(fp != NULL);
244-
printf("rank,var_name,version,mdata_time_ns,data_time_ns\n");
284+
char csv_filename[4096];
285+
// sprintf(csv_filename, "local_node_data_bandwidth_%d.csv", info.rank);
286+
// FILE* fp = redirect_stdout(csv_filename);
287+
// FILE* fp = fopen(csv_filename, "w+");
288+
// REQUIRE(fp != NULL);
289+
// printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n");
290+
double total_mdata_time = 0;
291+
double total_data_time = 0;
245292
for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) {
293+
long long int mdata_time_ns = 0;
294+
long long int data_time_ns = 0;
246295
data_time.resumeTime();
247296
// Using aget instead of get because dyad_get_data also allocates the buffer
248297
// Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available.
249-
rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1);
298+
rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns);
250299
data_time.pauseTime();
251300
REQUIRE (rc == dspaces_SUCCESS);
252301
free(file_data);
302+
total_mdata_time += NS_TO_SECS(mdata_time_ns);
303+
total_data_time += NS_TO_SECS(data_time_ns);
253304
}
254-
restore_stdout(fp);
255-
AGGREGATE_TIME(data);
256-
if (info.rank == 0) {
257-
printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n",
258-
info.comm_size, data_len*args.number_of_files,
259-
total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0);
260-
}
305+
// restore_stdout(fp);
306+
// AGGREGATE_TIME(data);
307+
gen_perf_print(data_len, total_mdata_time, total_data_time);
308+
rc = dspaces_fini(client);
309+
REQUIRE (rc == dspaces_SUCCESS);
310+
// fclose(fp);
311+
REQUIRE (posttest() == 0);
261312
}
262-
rc = dspaces_fini(client);
263-
REQUIRE (rc == dspaces_SUCCESS);
264-
REQUIRE (posttest() == 0);
265313
}

0 commit comments

Comments
 (0)