Skip to content

Commit

Permalink
feat: add table visualization in intermittents comments
Browse files Browse the repository at this point in the history
  • Loading branch information
ogiorgis committed Nov 18, 2024
1 parent 0b0a890 commit 8c952d2
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 4 deletions.
5 changes: 5 additions & 0 deletions tests/intermittents_commenter/expected_comment.text
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,10 @@ This is the #1 most frequent failure this week.
* b2g-emu-jb: 1
* debug: 1

## Table
| |B2G Emulator Image Build|
|---|:-:|
|b2g-emu-jb/debug|1|

## For more details, see:
https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2012-05-09&endday=2018-05-10&tree=all
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,14 @@ This is the #1 most frequent failure this week.
* windows7-32: 1
* debug: 1

## Table
| |B2G Emulator Image Build|Mochitest Browser Chrome|
|---|:-:|:-:|
|b2g-emu-jb/debug|1| |
|b2g-emu-jb/opt|1| |
|osx-10-6/debug| |1|
|osx-10-7/debug| |1|
|windows7-32/debug| |1|

## For more details, see:
https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2012-05-09&endday=2018-05-10&tree=all
15 changes: 15 additions & 0 deletions treeherder/intermittents_commenter/comment.template
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,20 @@ This is the #{{rank}} most frequent failure this week.{% endif %}
* {{ build }}: {{ count }}
{%- endfor %}
{% endfor %}
## Table
| |
{%- for variant in test_variants -%}
{{variant}}|
{%- endfor %}
|---|
{%- for test_variant in test_variants -%}
:-:|
{%- endfor %}
{% for platform_and_build, test_by_variant in test_suites.items() -%}
|{{platform_and_build}}|
{%- for variant in test_variants -%}
{{test_by_variant.get(variant, " ")}}|
{%- endfor %}
{% endfor %}
## For more details, see:
https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug={{bug_id}}&startday={{startday}}&endday={{endday}}&tree=all
38 changes: 34 additions & 4 deletions treeherder/intermittents_commenter/commenter.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def generate_bug_changes(self, startday, endday, alt_startday, alt_endday):
the appropriate threshold) and potentially an updated whiteboard
or priority status."""

bug_stats, bug_ids = self.get_bug_stats(startday, endday)
bug_stats, bug_ids, test_variants = self.get_bug_stats(startday, endday)
alt_date_bug_totals = self.get_alt_date_bug_totals(alt_startday, alt_endday, bug_ids)
test_run_count = self.get_test_runs(startday, endday)

Expand Down Expand Up @@ -103,6 +103,8 @@ def generate_bug_changes(self, startday, endday, alt_startday, alt_endday):
failure_rate=round(counts["total"] / float(test_run_count), 3),
repositories=counts["per_repository"],
platforms=counts["per_platform"],
test_variants=sorted(list(test_variants)),
test_suites=counts["test_suite_per_platform_and_build"],
counts=counts,
startday=startday,
endday=endday.split()[0],
Expand Down Expand Up @@ -277,6 +279,20 @@ def get_bug_stats(self, startday, endday):
"osx-10-10": 1,
}
},
"test_suite_per_platform_and_build": {
"windows10-64/debug" {
"mochitest-browser-chrome": 2,
"mochitest-browser-chrome-swr": 2,
},
"windows10-64/ccov" {
"mochitest-browser-chrome": 0,
"mochitest-browser-chrome-swr": 2,
},
"osx-10-10/debug": {
"mochitest-browser-chrome": 2,
"mochitest-browser-chrome-swr": 0,
},
},
"windows10-64": {
"debug": 30,
"ccov": 20,
Expand Down Expand Up @@ -307,24 +323,36 @@ def get_bug_stats(self, startday, endday):
"job__machine_platform__platform",
"bug_id",
"job__option_collection_hash",
"job__signature__job_type_name",
)
)

option_collection_map = OptionCollection.objects.get_option_collection_map()
bug_map = dict()
test_variants = set()

for bug in bugs:
platform = bug["job__machine_platform__platform"]
test_variant = bug["job__signature__job_type_name"].rsplit("-", 1)[0]
test_variants.add(test_variant)
repo = bug["job__repository__name"]
bug_id = bug["bug_id"]
build_type = option_collection_map.get(
bug["job__option_collection_hash"], "unknown build"
)

platform_and_build = f"{platform}/{build_type}"
if bug_id in bug_map:
bug_map[bug_id]["total"] += 1
bug_map[bug_id]["per_repository"][repo] += 1
bug_map[bug_id]["per_platform"][platform] += 1
if platform_and_build in bug_map[bug_id]["test_suite_per_platform_and_build"]:
bug_map[bug_id]["test_suite_per_platform_and_build"][platform_and_build][
test_variant
] += 1
else:
bug_map[bug_id]["test_suite_per_platform_and_build"][
platform_and_build
] = Counter([test_variant])
if bug_map[bug_id].get(platform):
bug_map[bug_id][platform][build_type] += 1
else:
Expand All @@ -334,10 +362,12 @@ def get_bug_stats(self, startday, endday):
bug_map[bug_id] = {}
bug_map[bug_id]["total"] = 1
bug_map[bug_id]["per_platform"] = Counter([platform])
bug_map[bug_id]["test_suite_per_platform_and_build"] = {
platform_and_build: Counter([test_variant])
}
bug_map[bug_id][platform] = Counter([build_type])
bug_map[bug_id]["per_repository"] = Counter([repo])

return bug_map, bug_ids
return bug_map, bug_ids, test_variants

def get_alt_date_bug_totals(self, startday, endday, bug_ids):
"""use previously fetched bug_ids to check for total failures
Expand Down

0 comments on commit 8c952d2

Please sign in to comment.