Table in comment

tests/conftest.py

@@ -1134,6 +1134,18 @@ def bug_data(eleven_jobs_stored, test_repository, test_push, bugs):
     }
 
 
+@pytest.fixture
+def bug_data_with_5_failures(eleven_jobs_stored, test_repository, test_push, bugs):
+    jobs = th_models.Job.objects.all().order_by("id")
+    bug_id = bugs[0].id
+    for job in jobs[:5]:
+        th_models.BugJobMap.create(job_id=job.id, bug_id=bug_id)
+
+    return {
+        "bug_id": bug_id,
+    }
+
+
 @pytest.fixture
 def test_run_data(bug_data):
     pushes = th_models.Push.objects.all()

tests/intermittents_commenter/expected_comment.text

@@ -1,13 +1,19 @@
+## Summary
 1 failures in 10 pushes (0.1 failures/push) were associated with this bug in the last 7 days.
 
 This is the #1 most frequent failure this week.
 
-Repository breakdown:
+## Repository breakdown:
 * mozilla-central: 1
 
-Platform and build breakdown:
+## Platform and build breakdown:
 * b2g-emu-jb: 1
   * debug: 1
 
-For more details, see:
+## Table
+| |**B2G Emulator Image Build**|
+|---|:-:|
+|**b2g-emu-jb/debug**|1|
+
+## For more details, see:
 https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2012-05-09&endday=2018-05-10&tree=all

tests/intermittents_commenter/expected_comment_with_5_failures.text

@@ -0,0 +1,30 @@
+## Summary
+5 failures in 10 pushes (0.5 failures/push) were associated with this bug in the last 7 days.
+
+This is the #1 most frequent failure this week.
+
+## Repository breakdown:
+* mozilla-central: 5
+
+## Platform and build breakdown:
+* b2g-emu-jb: 2
+  * debug: 1
+  * opt: 1
+* osx-10-6: 1
+  * debug: 1
+* osx-10-7: 1
+  * debug: 1
+* windows7-32: 1
+  * debug: 1
+
+## Table
+| |**B2G Emulator Image Build**|**Mochitest Browser Chrome**|
+|---|:-:|:-:|
+|**b2g-emu-jb/debug**|1| |
+|**b2g-emu-jb/opt**|1| |
+|**osx-10-6/debug**| |1|
+|**osx-10-7/debug**| |1|
+|**windows7-32/debug**| |1|
+
+## For more details, see:
+https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug=1&startday=2012-05-09&endday=2018-05-10&tree=all

tests/intermittents_commenter/test_commenter.py

@@ -42,3 +42,43 @@ def test_intermittents_commenter(bug_data):
     print(len(expected_comment))
     print(len(comment_params[0]["changes"]["comment"]["body"]))
     assert comment_params[0]["changes"]["comment"]["body"] == expected_comment
+
+
+@responses.activate
+def test_intermittents_commenter_with_failures(bug_data_with_5_failures):
+    startday = "2012-05-09"
+    endday = "2018-05-10"
+    alt_startday = startday
+    alt_endday = endday
+
+    process = Commenter(weekly_mode=True, dry_run=True)
+    params = {"include_fields": "product%2C+component%2C+priority%2C+whiteboard%2C+id"}
+    url = "{}/rest/bug?id={}&include_fields={}".format(
+        settings.BZ_API_URL, bug_data_with_5_failures["bug_id"], params["include_fields"]
+    )
+
+    content = {
+        "bugs": [
+            {
+                "component": "General",
+                "priority": "P3",
+                "product": "Testing",
+                "whiteboard": "[stockwell infra] [see summary at comment 92]",
+                "id": bug_data_with_5_failures["bug_id"],
+            }
+        ],
+        "faults": [],
+    }
+
+    responses.add(responses.Response(method="GET", url=url, json=content, status=200))
+
+    resp = process.fetch_bug_details(bug_data_with_5_failures["bug_id"])
+    assert resp == content["bugs"]
+
+    comment_params = process.generate_bug_changes(startday, endday, alt_startday, alt_endday)
+
+    with open("tests/intermittents_commenter/expected_comment_with_5_failures.text") as comment:
+        expected_comment = comment.read()
+    print(len(expected_comment))
+    print(len(comment_params[0]["changes"]["comment"]["body"]))
+    assert comment_params[0]["changes"]["comment"]["body"] == expected_comment

treeherder/intermittents_commenter/comment.template

@@ -1,35 +1,51 @@
-{{total}} failures in {{test_run_count}} pushes ({{failure_rate}} failures/push) were associated with this bug {% if weekly_mode %}in the last 7 days{% else %}yesterday{% endif %}. 
+## Summary
+{{total}} failures in {{test_run_count}} pushes ({{failure_rate}} failures/push) were associated with this bug {% if weekly_mode %}in the last 7 days{% else %}yesterday{% endif %}.
 
 {%- if rank %}
 
 This is the #{{rank}} most frequent failure this week.{% endif %}
 
 {%- if priority==1 %}
 
-** This failure happened more than 75 times this week! Resolving this bug is a very high priority. ** 
+**This failure happened more than 75 times this week! Resolving this bug is a very high priority.**
 
-** Try to resolve this bug as soon as possible. If unresolved for 1 week, the affected test(s) may be disabled. ** {% endif %}
+**Try to resolve this bug as soon as possible. If unresolved for 1 week, the affected test(s) may be disabled.** {% endif %}
 
 {%- if priority==2 %}
 
-** This failure happened more than 30 times this week! Resolving this bug is a high priority. **
+**This failure happened more than 30 times this week! Resolving this bug is a high priority.**
 
-** Try to resolve this bug as soon as possible. If unresolved for 2 weeks, the affected test(s) may be disabled. ** {% endif %}
+**Try to resolve this bug as soon as possible. If unresolved for 2 weeks, the affected test(s) may be disabled.** {% endif %}
 
 {%-if priority==3 %}
 
-** This test has failed more than 150 times in the last 21 days. It should be disabled until it can be fixed. ** {% endif %}
+**This test has failed more than 150 times in the last 21 days. It should be disabled until it can be fixed.** {% endif %}
 
-Repository breakdown:
+## Repository breakdown:
 {% for repository, count in repositories.items() -%}
 * {{repository}}: {{count}}
 {% endfor %}
-Platform and build breakdown:
+## Platform and build breakdown:
 {% for platform, count in platforms.items() -%}
 * {{platform}}: {{count}}
   {%- for build, count in counts[platform].items() %}
   * {{ build }}: {{ count }}
   {%- endfor %}
 {% endfor %}
-For more details, see:
+## Table
+| |
+{%- for variant in test_variants -%}
+  **{{variant}}**|
+{%- endfor %}
+|---|
+{%- for test_variant in test_variants -%}
+  :-:|
+{%- endfor %}
+{% for platform_and_build, test_by_variant in test_suites.items() -%}
+|**{{platform_and_build}}**|
+  {%- for variant in test_variants -%}
+    {{test_by_variant.get(variant, " ")}}|
+  {%- endfor %}
+{% endfor %}
+## For more details, see:
 https://treeherder.mozilla.org/intermittent-failures/bugdetails?bug={{bug_id}}&startday={{startday}}&endday={{endday}}&tree=all

treeherder/intermittents_commenter/commenter.py

@@ -56,12 +56,14 @@ def generate_bug_changes(self, startday, endday, alt_startday, alt_endday):
                 for bug in sorted(bug_stats.items(), key=lambda x: x[1]["total"], reverse=True)
             ][:50]
 
+        test_variants = set()
         for bug_id, counts in bug_stats.items():
             change_priority = None
             change_whiteboard = None
             priority = 0
             rank = top_bugs.index(bug_id) + 1 if self.weekly_mode and bug_id in top_bugs else None
 
+            test_variants |= bug_stats[bug_id]["test_variants"]
             if bug_info and bug_id in bug_info:
                 if self.weekly_mode:
                     priority = self.assign_priority(counts)
@@ -103,6 +105,8 @@ def generate_bug_changes(self, startday, endday, alt_startday, alt_endday):
                 failure_rate=round(counts["total"] / float(test_run_count), 3),
                 repositories=counts["per_repository"],
                 platforms=counts["per_platform"],
+                test_variants=sorted(test_variants),
+                test_suites=counts["test_suite_per_platform_and_build"],
                 counts=counts,
                 startday=startday,
                 endday=endday.split()[0],
@@ -277,6 +281,20 @@ def get_bug_stats(self, startday, endday):
                     "osx-10-10": 1,
                     }
                 },
+               "test_suite_per_platform_and_build": {
+                    "windows10-64/debug" {
+                        "mochitest-browser-chrome": 2,
+                        "mochitest-browser-chrome-swr": 2,
+                     },
+                     "windows10-64/ccov" {
+                        "mochitest-browser-chrome": 0,
+                        "mochitest-browser-chrome-swr": 2,
+                     },
+                    "osx-10-10/debug": {
+                        "mochitest-browser-chrome": 2,
+                        "mochitest-browser-chrome-swr": 0,
+                     },
+                },
                 "windows10-64": {
                     "debug": 30,
                     "ccov": 20,
@@ -301,11 +319,13 @@ def get_bug_stats(self, startday, endday):
         bugs = (
             BugJobMap.failures.by_date(startday, endday)
             .filter(bug_id__in=bug_ids)
+            .order_by("job__machine_platform__platform")
             .values(
                 "job__repository__name",
                 "job__machine_platform__platform",
                 "bug_id",
                 "job__option_collection_hash",
+                "job__signature__job_type_name",
             )
         )
 
@@ -314,28 +334,41 @@ def get_bug_stats(self, startday, endday):
 
         for bug in bugs:
             platform = bug["job__machine_platform__platform"]
+            test_variant = bug["job__signature__job_type_name"].rsplit("-", 1)[0]
             repo = bug["job__repository__name"]
             bug_id = bug["bug_id"]
             build_type = option_collection_map.get(
                 bug["job__option_collection_hash"], "unknown build"
             )
-
+            platform_and_build = f"{platform}/{build_type}"
             if bug_id in bug_map:
+                bug_map[bug_id]["test_variants"].add(test_variant)
                 bug_map[bug_id]["total"] += 1
                 bug_map[bug_id]["per_repository"][repo] += 1
                 bug_map[bug_id]["per_platform"][platform] += 1
+                if platform_and_build in bug_map[bug_id]["test_suite_per_platform_and_build"]:
+                    bug_map[bug_id]["test_suite_per_platform_and_build"][platform_and_build][
+                        test_variant
+                    ] += 1
+                else:
+                    bug_map[bug_id]["test_suite_per_platform_and_build"][
+                        platform_and_build
+                    ] = Counter([test_variant])
                 if bug_map[bug_id].get(platform):
                     bug_map[bug_id][platform][build_type] += 1
                 else:
                     bug_map[bug_id][platform] = Counter([build_type])
 
             else:
                 bug_map[bug_id] = {}
+                bug_map[bug_id]["test_variants"] = set([test_variant])
                 bug_map[bug_id]["total"] = 1
                 bug_map[bug_id]["per_platform"] = Counter([platform])
+                bug_map[bug_id]["test_suite_per_platform_and_build"] = {
+                    platform_and_build: Counter([test_variant])
+                }
                 bug_map[bug_id][platform] = Counter([build_type])
                 bug_map[bug_id]["per_repository"] = Counter([repo])
-
         return bug_map, bug_ids
 
     def get_alt_date_bug_totals(self, startday, endday, bug_ids):