Bug 1831941 - Ingest replicates in a new PerformanceDatumReplicate table. (#7693)

* Bug XXX - Ingest replicates in a new PerformanceDatumReplicate table.

* Changes

* changes

* changes

* lint fixes

Author: gmierz

treeherder/etl/perf.py

@@ -2,16 +2,17 @@
 import logging
 from datetime import datetime
 from hashlib import sha1
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 import simplejson as json
 
 from django.conf import settings
 from treeherder.log_parser.utils import validate_perf_data
-from treeherder.model.models import Job, OptionCollection
+from treeherder.model.models import Job, OptionCollection, Repository
 from treeherder.perf.models import (
     MultiCommitDatum,
     PerformanceDatum,
+    PerformanceDatumReplicate,
     PerformanceFramework,
     PerformanceSignature,
 )
@@ -117,6 +118,18 @@ def _test_should_alert_based_on(
     )
 
 
+def _test_should_gather_replicates_based_on(
+    repository: Repository, replicates: Optional[List] = None
+) -> bool:
+    """
+    Determine if we should gather/ingest replicates. Currently, it's
+    only available on the try branch. Some tests also don't have replicates
+    available as it's not a required field in our performance artifact
+    schema.
+    """
+    return replicates and len(replicates) > 0 and repository.name in ("try",)
+
+
 def _load_perf_datum(job: Job, perf_datum: dict):
     validate_perf_data(perf_datum)
 
@@ -271,6 +284,25 @@ def _load_perf_datum(job: Job, perf_datum: dict):
                 push_timestamp=deduced_timestamp,
                 defaults={'value': value[0], 'application_version': application_version},
             )
+
+            if _test_should_gather_replicates_based_on(
+                job.repository, subtest.get("replicates", [])
+            ):
+                try:
+                    # Add the replicates to the PerformanceDatumReplicate table, and
+                    # catch and ignore any exceptions that are produced here so we don't
+                    # impact the standard workflow
+                    PerformanceDatumReplicate.objects.bulk_create(
+                        [
+                            PerformanceDatumReplicate(
+                                value=replicate, performance_datum=subtest_datum
+                            )
+                            for replicate in subtest["replicates"]
+                        ]
+                    )
+                except Exception as e:
+                    logger.info(f"Failed to ingest replicates for datum {subtest_datum}: {e}")
+
             if subtest_datum.should_mark_as_multi_commit(is_multi_commit, datum_created):
                 # keep a register with all multi commit perf data
                 MultiCommitDatum.objects.create(perf_datum=subtest_datum)

treeherder/perf/migrations/0049_performancedatumreplicate.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.0.8 on 2023-05-08 17:59
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('perf', '0048_performancedatum_application_version'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PerformanceDatumReplicate',
+            fields=[
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
+                ('value', models.FloatField()),
+                (
+                    'performance_datum',
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to='perf.performancedatum'
+                    ),
+                ),
+            ],
+            options={
+                'db_table': 'performance_datum_replicate',
+            },
+        ),
+    ]

treeherder/perf/models.py

@@ -105,7 +105,7 @@ def _get_alert_change_type(alert_change_type_input):
         Maps a schema-specified alert change type to the internal index
         value
         """
-        for (idx, enum_val) in PerformanceSignature.ALERT_CHANGE_TYPES:
+        for idx, enum_val in PerformanceSignature.ALERT_CHANGE_TYPES:
             if enum_val == alert_change_type_input:
                 return idx
         return None
@@ -227,6 +227,15 @@ def __str__(self):
         return "{} {}".format(self.value, self.push_timestamp)
 
 
+class PerformanceDatumReplicate(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    performance_datum = models.ForeignKey(PerformanceDatum, on_delete=models.CASCADE)
+    value = models.FloatField()
+
+    class Meta:
+        db_table = 'performance_datum_replicate'
+
+
 class MultiCommitDatum(models.Model):
     perf_datum = models.OneToOneField(
         PerformanceDatum,