Skip to content

Commit beedb9b

Browse files
bwbroersmabwbroersma
committed
Fix #1246 - Improve Hall of Fame task
1 parent cef300d commit beedb9b

File tree

6 files changed

+253
-91
lines changed

6 files changed

+253
-91
lines changed

‎checks/migrations/0019_hall_of_fame_plus_triggers.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
# Partly generated by Django 4.2.20 on 2025-05-25 15:24 together with manual RunSQL
2+
3+
from django.db import migrations, models
4+
import django.db.models.deletion
5+
import pgtrigger.compiler
6+
import pgtrigger.migrations
7+
8+
9+
class Migration(migrations.Migration):
10+
dependencies = [
11+
("checks", "0018_domaintesttls_caa_records"),
12+
]
13+
14+
operations = [
15+
# Note db_index is False on the ForeignKey to prevent extra indices that are not needed
16+
# AutoField has to be primary key in Django, to solve this manually alter SQL this field
17+
# see https://github.com/django/django/blob/787f3130f751283140fe2be8188eb5299552232d/django/db/models/fields/__init__.py#L2801
18+
# The primary key on domain is added in SQL to prevent a _like index
19+
migrations.CreateModel(
20+
name="Fame",
21+
fields=[
22+
("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
23+
("domain", models.CharField(max_length=255)),
24+
(
25+
"site_report",
26+
models.ForeignKey(
27+
db_index=False,
28+
null=True,
29+
on_delete=django.db.models.deletion.CASCADE,
30+
to="checks.domaintestreport",
31+
),
32+
),
33+
("site_report_timestamp", models.DateTimeField(null=True)),
34+
(
35+
"mail_report",
36+
models.ForeignKey(
37+
db_index=False,
38+
null=True,
39+
on_delete=django.db.models.deletion.CASCADE,
40+
to="checks.mailtestreport",
41+
),
42+
),
43+
("mail_report_timestamp", models.DateTimeField(null=True)),
44+
],
45+
),
46+
pgtrigger.migrations.AddTrigger(
47+
model_name="domaintestreport",
48+
trigger=pgtrigger.compiler.Trigger(
49+
name="update_fame_on_site_report",
50+
sql=pgtrigger.compiler.UpsertTriggerSql(
51+
func="""
52+
IF NEW.score IS NULL THEN
53+
-- DO NOTHING
54+
ELSIF NEW.score = 100 THEN
55+
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
56+
VALUES (NEW.domain, NEW.id, NEW.timestamp, NULL, NULL)
57+
ON CONFLICT (domain)
58+
DO UPDATE SET site_report_id = NEW.id, site_report_timestamp = NEW.timestamp;
59+
ELSE
60+
MERGE INTO ONLY checks_fame c1
61+
USING checks_fame c2 ON c1.domain = c2.domain AND c1.domain = NEW.domain
62+
WHEN NOT MATCHED THEN
63+
DO NOTHING
64+
WHEN MATCHED AND c1.mail_report_id IS NOT NULL THEN
65+
UPDATE SET site_report_id = NULL, site_report_timestamp = NULL
66+
WHEN MATCHED AND c1.mail_report_id IS NULL THEN
67+
DELETE;
68+
END IF;
69+
RETURN NEW;
70+
""",
71+
hash="b4f792b06123914de71b57669c202a19b04e9e9c",
72+
operation='INSERT OR UPDATE OF "score"',
73+
pgid="pgtrigger_update_fame_on_site_report_e4fdc",
74+
table="checks_domaintestreport",
75+
when="AFTER",
76+
),
77+
),
78+
),
79+
pgtrigger.migrations.AddTrigger(
80+
model_name="mailtestreport",
81+
trigger=pgtrigger.compiler.Trigger(
82+
name="update_fame_on_mail_report",
83+
sql=pgtrigger.compiler.UpsertTriggerSql(
84+
func="""
85+
IF NEW.score IS NULL THEN
86+
-- DO NOTHING
87+
ELSIF NEW.score = 100 THEN
88+
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
89+
VALUES (NEW.domain, NULL, NULL, NEW.id, NEW.timestamp)
90+
ON CONFLICT (domain)
91+
DO UPDATE SET mail_report_id = NEW.id, mail_report_timestamp = NEW.timestamp;
92+
ELSE
93+
MERGE INTO ONLY checks_fame c1
94+
USING checks_fame c2 ON c1.domain = c2.domain AND c1.domain = NEW.domain
95+
WHEN NOT MATCHED THEN
96+
DO NOTHING
97+
WHEN MATCHED AND c1.site_report_id IS NOT NULL THEN
98+
UPDATE SET mail_report_id = NULL, mail_report_timestamp = NULL
99+
WHEN MATCHED AND c1.site_report_id IS NULL THEN
100+
DELETE;
101+
END IF;
102+
RETURN NEW;
103+
""",
104+
hash="707aefc7a83dd041dd815511f1d1cf7e8f84f944",
105+
operation='INSERT OR UPDATE OF "score"',
106+
pgid="pgtrigger_update_fame_on_mail_report_b3a27",
107+
table="checks_mailtestreport",
108+
when="AFTER",
109+
),
110+
),
111+
),
112+
migrations.RunSQL(
113+
sql=[
114+
'ALTER TABLE "checks_fame" DROP CONSTRAINT "checks_fame_pkey";',
115+
'ALTER TABLE "checks_fame" ADD PRIMARY KEY ("domain");',
116+
"""
117+
WITH
118+
site_fame AS (
119+
SELECT domain, id AS site_report_id, timestamp AS site_report_timestamp FROM (
120+
SELECT domain, score, id, timestamp, rank() OVER (PARTITION BY domain ORDER BY id DESC) FROM checks_domaintestreport
121+
) alias WHERE rank = 1 AND score = 100),
122+
mail_fame AS (
123+
SELECT domain, id AS mail_report_id, timestamp AS mail_report_timestamp FROM (
124+
SELECT domain, score, id, timestamp, rank() OVER (PARTITION BY domain ORDER BY id DESC) FROM checks_mailtestreport
125+
) alias WHERE rank = 1 AND score = 100)
126+
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
127+
SELECT * FROM site_fame FULL OUTER JOIN mail_fame USING (domain);
128+
""",
129+
],
130+
reverse_sql=[
131+
'DELETE FROM "checks_fame";',
132+
'ALTER TABLE "checks_fame" DROP CONSTRAINT "checks_fame_pkey";',
133+
'ALTER TABLE "checks_fame" ADD PRIMARY KEY ("id");',
134+
],
135+
state_operations=[
136+
migrations.AddConstraint("Fame", models.UniqueConstraint(fields=["domain"], name="checks_fame_pkey")),
137+
migrations.AddIndex("Fame", models.Index(fields=["domain"], name="checks_fame_pkey")),
138+
],
139+
),
140+
]

‎checks/models.py‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from django.utils import timezone
1111
from enumfields import Enum as LabelEnum
1212
from enumfields import EnumField, EnumIntegerField
13+
import pgtrigger
1314

1415

1516
class ListField(models.TextField):
@@ -947,6 +948,34 @@ def __dir__(self):
947948
class Meta:
948949
app_label = "checks"
949950

951+
triggers = [
952+
pgtrigger.Trigger(
953+
name="update_fame_on_site_report",
954+
when=pgtrigger.After,
955+
operation=pgtrigger.Insert | pgtrigger.UpdateOf("score"),
956+
func="""
957+
IF NEW.score IS NULL THEN
958+
-- DO NOTHING
959+
ELSIF NEW.score = 100 THEN
960+
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
961+
VALUES (NEW.domain, NEW.id, NEW.timestamp, NULL, NULL)
962+
ON CONFLICT (domain)
963+
DO UPDATE SET site_report_id = NEW.id, site_report_timestamp = NEW.timestamp;
964+
ELSE
965+
MERGE INTO ONLY checks_fame c1
966+
USING checks_fame c2 ON c1.domain = c2.domain AND c1.domain = NEW.domain
967+
WHEN NOT MATCHED THEN
968+
DO NOTHING
969+
WHEN MATCHED AND c1.mail_report_id IS NOT NULL THEN
970+
UPDATE SET site_report_id = NULL, site_report_timestamp = NULL
971+
WHEN MATCHED AND c1.mail_report_id IS NULL THEN
972+
DELETE;
973+
END IF;
974+
RETURN NEW;
975+
""",
976+
),
977+
]
978+
950979

951980
###
952981
# Mail test
@@ -1093,6 +1122,56 @@ def __dir__(self):
10931122
class Meta:
10941123
app_label = "checks"
10951124

1125+
triggers = [
1126+
pgtrigger.Trigger(
1127+
name="update_fame_on_mail_report",
1128+
when=pgtrigger.After,
1129+
operation=pgtrigger.Insert | pgtrigger.UpdateOf("score"),
1130+
func="""
1131+
IF NEW.score IS NULL THEN
1132+
-- DO NOTHING
1133+
ELSIF NEW.score = 100 THEN
1134+
INSERT INTO checks_fame (domain, site_report_id, site_report_timestamp, mail_report_id, mail_report_timestamp)
1135+
VALUES (NEW.domain, NULL, NULL, NEW.id, NEW.timestamp)
1136+
ON CONFLICT (domain)
1137+
DO UPDATE SET mail_report_id = NEW.id, mail_report_timestamp = NEW.timestamp;
1138+
ELSE
1139+
MERGE INTO ONLY checks_fame c1
1140+
USING checks_fame c2 ON c1.domain = c2.domain AND c1.domain = NEW.domain
1141+
WHEN NOT MATCHED THEN
1142+
DO NOTHING
1143+
WHEN MATCHED AND c1.site_report_id IS NOT NULL THEN
1144+
UPDATE SET mail_report_id = NULL, mail_report_timestamp = NULL
1145+
WHEN MATCHED AND c1.site_report_id IS NULL THEN
1146+
DELETE;
1147+
END IF;
1148+
RETURN NEW;
1149+
""",
1150+
),
1151+
]
1152+
1153+
1154+
class Fame(models.Model):
1155+
id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")
1156+
domain = models.CharField(max_length=255)
1157+
site_report = models.ForeignKey(DomainTestReport, null=True, on_delete=models.CASCADE, db_index=False)
1158+
site_report_timestamp = models.DateTimeField(null=True)
1159+
mail_report = models.ForeignKey(MailTestReport, null=True, on_delete=models.CASCADE, db_index=False)
1160+
mail_report_timestamp = models.DateTimeField(null=True)
1161+
1162+
def __dir__(self):
1163+
return ["domain", "site_report", "site_report_timestamp", "mail_report", "mail_report_timestamp"]
1164+
1165+
class Meta:
1166+
app_label = "checks"
1167+
1168+
constraints = [
1169+
models.UniqueConstraint(fields=["domain"], name="checks_fame_pkey"),
1170+
]
1171+
indexes = [
1172+
models.Index(fields=["domain"], name="checks_fame_pkey"),
1173+
]
1174+
10961175

10971176
class BatchUser(models.Model):
10981177
"""

‎checks/tasks/update.py‎

Lines changed: 25 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from django.core.cache import cache
66
from django.db import transaction
77

8-
from checks.models import DomainTestReport, MailTestReport
8+
from checks.models import Fame
99
from interface import redis_id
1010
from interface.batch import util
1111

@@ -19,85 +19,16 @@ def __init__(self, domain):
1919
self.web_permalink = None
2020
self.mail_timestamp = None
2121
self.mail_permalink = None
22-
self.mail_nomx = None
2322

2423
def __str__(self):
2524
return f"""------- {self.domain}
2625
web_timestamp: {self.web_timestamp}
2726
web_permalink: {self.web_permalink}
2827
mail_timestamp: {self.mail_timestamp}
2928
mail_permalink: {self.mail_permalink}
30-
mail_nomx: {self.mail_nomx}
3129
"""
3230

3331

34-
def _create_hof_entry(hof, domain_name):
35-
"""
36-
Create an entry in the Hall of Fame.
37-
38-
"""
39-
if domain_name in hof:
40-
return hof[domain_name]
41-
hof[domain_name] = HOFEntry(domain_name)
42-
return hof[domain_name]
43-
44-
45-
def _update_web_entry(hof, domain_name, report_id, timestamp):
46-
"""
47-
Update a web entry in the Hall of Fame.
48-
49-
"""
50-
entry = _create_hof_entry(hof, domain_name)
51-
entry.web_timestamp = timestamp
52-
entry.web_permalink = f"/site/{domain_name}/{report_id}/"
53-
54-
55-
def _update_mail_entry(hof, domain_name, report_id, timestamp):
56-
"""
57-
Update a mail entry in the Hall of Fame.
58-
59-
"""
60-
entry = _create_hof_entry(hof, domain_name)
61-
entry.mail_timestamp = timestamp
62-
entry.mail_permalink = f"/mail/{domain_name}/{report_id}/"
63-
report = MailTestReport.objects.get(id=report_id)
64-
ipv6_report = report.ipv6.report
65-
if not isinstance(ipv6_report, dict):
66-
return
67-
entry.mail_nomx = ipv6_report["mx_aaaa"]["verdict"] == "detail mail ipv6 mx-AAAA verdict other"
68-
69-
70-
def _populate_HOF(hof, model, entry_creation):
71-
"""
72-
Find entries that qualify for the Hall of Fame.
73-
74-
"""
75-
previousname = None
76-
previousscore = 0
77-
previoustimestamp = None
78-
previousreportid = None
79-
for report in model.objects.all().order_by("domain", "timestamp"):
80-
if previousname != report.domain and previousname is not None:
81-
if previousscore >= 100:
82-
entry_creation(hof, previousname, previousreportid, previoustimestamp)
83-
previousname = report.domain
84-
previousscore = report.score or 0
85-
previoustimestamp = report.timestamp
86-
previousreportid = report.id
87-
88-
else:
89-
report_score = report.score or 0
90-
if report_score != previousscore:
91-
previoustimestamp = report.timestamp
92-
previousname = report.domain
93-
previousreportid = report.id
94-
previousscore = report_score
95-
96-
# Last domain name.
97-
if previousscore >= 100:
98-
entry_creation(hof, previousname, previousreportid, previoustimestamp)
99-
100-
10132
@transaction.atomic
10233
def _update_hof():
10334
"""
@@ -108,28 +39,33 @@ def _update_hof():
10839
test scored 100%.
10940
11041
"""
111-
hof = dict()
112-
for model, entry_creation in ((DomainTestReport, _update_web_entry), (MailTestReport, _update_mail_entry)):
113-
_populate_HOF(hof, model, entry_creation)
114-
11542
champions = []
11643
web = []
11744
mail = []
118-
for entry in hof.values():
119-
is_web = False
120-
is_mail = False
121-
if entry.web_permalink:
122-
web.append({"permalink": entry.web_permalink, "domain": entry.domain, "timestamp": entry.web_timestamp})
123-
is_web = True
124-
if entry.mail_permalink:
125-
mail.append({"permalink": entry.mail_permalink, "domain": entry.domain, "timestamp": entry.mail_timestamp})
126-
is_mail = True
127-
if is_web and is_mail:
128-
timestamp = entry.mail_timestamp
129-
permalink = entry.mail_permalink
130-
if entry.web_timestamp > entry.mail_timestamp:
131-
timestamp = entry.web_timestamp
132-
permalink = entry.web_permalink
45+
46+
for entry in Fame.objects.all().iterator():
47+
if entry.site_report_id is not None:
48+
web.append(
49+
{
50+
"permalink": f"/site/{entry.domain}/{entry.site_report_id}/",
51+
"domain": entry.domain,
52+
"timestamp": entry.site_report_timestamp,
53+
}
54+
)
55+
if entry.mail_report_id is not None:
56+
mail.append(
57+
{
58+
"permalink": f"/mail/{entry.domain}/{entry.mail_report_id}/",
59+
"domain": entry.domain,
60+
"timestamp": entry.mail_report_timestamp,
61+
}
62+
)
63+
if entry.site_report_id is not None and entry.mail_report_id is not None:
64+
timestamp = entry.mail_report_timestamp
65+
permalink = f"/mail/{entry.domain}/{entry.mail_report_id}/"
66+
if entry.site_report_timestamp > entry.mail_report_timestamp:
67+
timestamp = entry.site_report_timestamp
68+
permalink = f"/site/{entry.domain}/{entry.site_report_id}/"
13369
champions.append({"permalink": permalink, "domain": entry.domain, "timestamp": timestamp})
13470
champions = sorted(champions, key=lambda x: x["timestamp"], reverse=True)
13571
web = sorted(web, key=lambda x: x["timestamp"], reverse=True)

0 commit comments

Comments
 (0)