@@ -238,26 +238,34 @@ def _check_osu_benchmarks_results(test_datadir, instance, mpi_version, benchmark
238
238
logging .info (output )
239
239
# Check avg latency for all packet sizes
240
240
failures = 0
241
- for packet_size , latency in re .findall (r"(\d+)\s+(\d+)\." , output ):
241
+ for packet_size , value in re .findall (r"(\d+)\s+(\d+)\." , output ):
242
242
with open (
243
243
str (test_datadir / "osu_benchmarks" / "results" / instance / mpi_version / benchmark_name ), encoding = "utf-8"
244
244
) as result :
245
245
previous_result = re .search (rf"{ packet_size } \s+(\d+)\." , result .read ()).group (1 )
246
246
247
- # Use a tolerance of 10us for 2 digits values.
248
- # For 3+ digits values use a 20% tolerance, except for the higher-variance latency benchmark.
249
- if len (previous_result ) <= 2 :
250
- accepted_tolerance = 10
247
+ if benchmark_name == "osu_bibw" :
248
+ # Invert logic because osu_bibw is in MB/s
249
+ tolerated_value = float (previous_result ) - ( float ( previous_result ) * 0.2 )
250
+ is_failure = int ( value ) < tolerated_value
251
251
else :
252
- multiplier = 0.3 if benchmark_name == "osu_latency" else 0.2
253
- accepted_tolerance = float (previous_result ) * multiplier
254
- tolerated_latency = float (previous_result ) + accepted_tolerance
252
+ # Use a tolerance of 10us for 2 digits values.
253
+ # For 3+ digits values use a 20% tolerance, except for the higher-variance latency benchmark.
254
+ if len (previous_result ) <= 2 :
255
+ accepted_tolerance = 10
256
+ else :
257
+ multiplier = 0.3 if benchmark_name == "osu_latency" else 0.2
258
+ accepted_tolerance = float (previous_result ) * multiplier
259
+ tolerated_value = float (previous_result ) + accepted_tolerance
260
+
261
+ is_failure = int (value ) > tolerated_value
255
262
256
263
message = (
257
264
f"{ mpi_version } - { benchmark_name } - packet size { packet_size } : "
258
- f"tolerated: { tolerated_latency } , current: { latency } "
265
+ f"tolerated: { tolerated_value } , current: { value } "
259
266
)
260
- if int (latency ) > tolerated_latency :
267
+
268
+ if is_failure :
261
269
failures = failures + 1
262
270
logging .error (message )
263
271
else :
0 commit comments