Skip to content

Commit 1ae80ac

Browse files
francesco-giordanoeantonin
authored andcommitted
Fix benchmark result check in case of osu_bibw test
Signed-off-by: Francesco Giordano <[email protected]>
1 parent e9f0688 commit 1ae80ac

File tree

1 file changed

+18
-10
lines changed

1 file changed

+18
-10
lines changed

tests/integration-tests/tests/efa/test_efa.py

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -238,26 +238,34 @@ def _check_osu_benchmarks_results(test_datadir, instance, mpi_version, benchmark
238238
logging.info(output)
239239
# Check avg latency for all packet sizes
240240
failures = 0
241-
for packet_size, latency in re.findall(r"(\d+)\s+(\d+)\.", output):
241+
for packet_size, value in re.findall(r"(\d+)\s+(\d+)\.", output):
242242
with open(
243243
str(test_datadir / "osu_benchmarks" / "results" / instance / mpi_version / benchmark_name), encoding="utf-8"
244244
) as result:
245245
previous_result = re.search(rf"{packet_size}\s+(\d+)\.", result.read()).group(1)
246246

247-
# Use a tolerance of 10us for 2 digits values.
248-
# For 3+ digits values use a 20% tolerance, except for the higher-variance latency benchmark.
249-
if len(previous_result) <= 2:
250-
accepted_tolerance = 10
247+
if benchmark_name == "osu_bibw":
248+
# Invert logic because osu_bibw is in MB/s
249+
tolerated_value = float(previous_result) - (float(previous_result) * 0.2)
250+
is_failure = int(value) < tolerated_value
251251
else:
252-
multiplier = 0.3 if benchmark_name == "osu_latency" else 0.2
253-
accepted_tolerance = float(previous_result) * multiplier
254-
tolerated_latency = float(previous_result) + accepted_tolerance
252+
# Use a tolerance of 10us for 2 digits values.
253+
# For 3+ digits values use a 20% tolerance, except for the higher-variance latency benchmark.
254+
if len(previous_result) <= 2:
255+
accepted_tolerance = 10
256+
else:
257+
multiplier = 0.3 if benchmark_name == "osu_latency" else 0.2
258+
accepted_tolerance = float(previous_result) * multiplier
259+
tolerated_value = float(previous_result) + accepted_tolerance
260+
261+
is_failure = int(value) > tolerated_value
255262

256263
message = (
257264
f"{mpi_version} - {benchmark_name} - packet size {packet_size}: "
258-
f"tolerated: {tolerated_latency}, current: {latency}"
265+
f"tolerated: {tolerated_value}, current: {value}"
259266
)
260-
if int(latency) > tolerated_latency:
267+
268+
if is_failure:
261269
failures = failures + 1
262270
logging.error(message)
263271
else:

0 commit comments

Comments
 (0)