Merge branch 'internetstandards:main' into feature/881-tlsrpt

Changelog.md

@@ -1,5 +1,18 @@
 # Change Log
 
+## 1.8.5
+
+Release 1.8.5 contains a hotfix for the [sectxt library failing on leap days](https://github.com/DigitalTrustCenter/sectxt/issues/66).
+
+## 1.8.4
+
+Release 1.8.4:
+
+- Updates unbound to 1.19.1-internetnl to fix CVE-2023-50387 and CVE-2023-50868.
+- Restricts HTTPS redirects to the same domain, no longer allowing directions to a subdomain first (#1208).
+- Updates a number of other dependencies.
+- Fixes an issue where certbot renewals were not correctly run.
+
 ## 1.8.3
 
 Release 1.8.3 fixes an issue where HSTS and CSP headers were missing from he www-subdomain of the main domain (#1210, #1211).

Makefile

@@ -4,7 +4,6 @@ PY?=python
 TAR?=0
 
 BINDIR=bin
-POFILESEXEC=$(BINDIR)/pofiles.py
 FRONTENDEXEC=$(BINDIR)/frontend.py
 
 REMOTEDATADIR=remote_data
@@ -26,13 +25,6 @@ mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
 current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path))))
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 
-ifeq ($(TAR), 0)
-	POFILES_TAR_ARGS=to_tar
-else
-	POFILES_TAR_ARGS=from_tar
-	POFILES_TAR_ARGS+=$(TAR)
-endif
-
 pysrcdirs = internetnl tests interface checks integration_tests
 pysrc = $(shell find ${pysrcdirs} -name \*.py)
 
@@ -45,9 +37,8 @@ help:
 	@echo 'Makefile for internet.nl'
 	@echo ''
 	@echo 'Usage:'
-	@echo '   make translations                          combine the translation files to Django PO files'
-	@echo '   make translations_tar                      create a tar from the translations'
-	@echo '   make translations_tar TAR=<tar.gz file>    read the tar and update the translations'
+	@echo '   make update_content                        update the translation files from content repo.'
+	@echo '                                              Optional branch=x to use a specific content repo branch.'
 	@echo '   make frontend                              (re)generate CSS and Javascript'
 	@echo '   make update_padded_macs                    update padded MAC information'
 	@echo '   make update_cert_fingerprints              update certificate fingerpint information'
@@ -69,25 +60,18 @@ frontend:
 	. .venv/bin/activate && ${_env} python3 manage.py collectstatic --no-input
 	. .venv/bin/activate && ${_env} python3 manage.py api_generate_doc
 
+	${DOCKER_COMPOSE_TOOLS_CMD} run --rm tools bin/lint.sh ${pysrcdirs}
 
-translate_content_to_main:
-	# Note: you may need to run this a few times to get rid of the access denied errors...
-	# This retrieves the content from the content repository and merges it with the .po files of this repo.
-	# The procedure is detailed at: https://github.com/internetstandards/Internet.nl_content/blob/master/.README.md
+branch ?= main
+update_content:
+    # This retrieves the content from the content repository and merges it with the .po files of this repo.
+    # The procedure is detailed at: https://github.com/internetstandards/Internet.nl_content/blob/master/.README.md
 	rm -rf tmp/locale_files/
 	rm -f tmp/content_repo.tar.gz
-	git clone [email protected]:internetstandards/Internet.nl_content/ tmp/locale_files/
-
-	# If you need a specific branch people are working on:
-	# git clone -b news-item_PLIS-meeting_on_IPv6 https://github.com/internetstandards/Internet.nl_content/ tmp/locale_files/
-
-	# change dir to tmp to prevent the /tmp dir being mentioned in the resulting tar file.
-	cd tmp && tar zcvf content_repo.tar.gz locale_files/*
-	${MAKE} translations_tar TAR=tmp/content_repo.tar.gz
-	${MAKE} translations
-	. .venv/bin/activate && ${_env} python3 manage.py compilemessages --ignore=.venv
-	# Purposefully _not_ deleting things in the tmp dir so it allows inspection after execution.
-
+	mkdir -p tmp/locale_files/
+	git clone -b $(branch) [email protected]:internetstandards/Internet.nl_content/ tmp/locale_files/
+	${DOCKER_COMPOSE_TOOLS_CMD} run --rm tools bin/update_translations.sh
+	rm -rf tmp/locale_files
 
 update_padded_macs:
 	chmod +x $(MACSDIR)/update-macs.sh
@@ -651,12 +635,5 @@ documentation-images: ${images}
 documentation/images/%.png: documentation/images/%.py | ${nwdiag}
 	docker run -it --rm -v "$${PWD}/$(@D)/":/$(@D) -w /$(@D) gtramontina/diagrams:0.23.1 $(<F)
 
-batch-api-add-user docker-compose-batch-api-add-user: name=${username}
-batch-api-add-user docker-compose-batch-api-add-user: organization=internetnl
-batch-api-add-user docker-compose-batch-api-add-user: email=${username}@example.com
-batch-api-add-user docker-compose-batch-api-add-user:
-	${DOCKER_COMPOSE_CMD} exec app ./manage.py api_users register -u ${username} -n ${name} -o ${organization} -e ${email}
-	${DOCKER_COMPOSE_CMD} exec webserver htpasswd -b /etc/nginx/htpasswd/batch_api.htpasswd ${username} ${password}
-
 test-%: env=test
 test-up test-down test-build test-stop: test-%: %

bin/update_translations.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+tar --strip-components=1 -cf tmp/content_repo.tar.gz locale_files/*
+python3 bin/pofiles.py from_tar tmp/content_repo.tar.gz
+
+# to_django is performed in Dockerfile

checks/categories.py

@@ -539,6 +539,10 @@ def result_bad(self):
         self._status(STATUS_FAIL)
         self.verdict = "detail web ipv6 web-ipv46 verdict bad"
 
+    def result_notice_status_code(self):
+        self._status(STATUS_NOTICE)
+        self.verdict = "detail web ipv6 web-ipv46 verdict notice-status-code"
+
     def result_notice(self):
         self._status(STATUS_NOTICE)
         self.verdict = "detail web ipv6 web-ipv46 verdict notice"

checks/http_client.py

@@ -12,6 +12,7 @@
 from checks.tasks import SetupUnboundContext
 from checks.tasks.tls_connection import DEFAULT_TIMEOUT
 from checks.tasks.tls_connection_exceptions import NoIpError
+from django.conf import settings
 from interface.views.shared import ub_resolve_with_timeout
 from internetnl import log
 
@@ -59,7 +60,7 @@ def http_get(
 
     if not headers:
         headers = {}
-    headers["User-Agent"] = "internetnl/1.0"
+    headers["User-Agent"] = settings.USER_AGENT
     if not session:
         session = requests.session()
 

checks/migrations/0015_auto_20240212_1616.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.23 on 2024-02-12 16:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("checks", "0014_auto_20230804_0855"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="batchuser",
+            name="email",
+        ),
+        migrations.RemoveField(
+            model_name="batchuser",
+            name="name",
+        ),
+        migrations.RemoveField(
+            model_name="batchuser",
+            name="organization",
+        ),
+    ]

checks/models.py

@@ -1067,19 +1067,13 @@ class Meta:
 
 class BatchUser(models.Model):
     """
-    Users allowed to run batch tests.
-
-    .. note:: Must be in sync with the web authorization scheme.
-
+    Users allowed to run batch tests. Automatically created when encountering new users.
     """
 
     username = models.CharField(unique=True, max_length=255)
-    name = models.CharField(max_length=255)
-    organization = models.CharField(max_length=255)
-    email = models.EmailField(max_length=255)
 
     def __dir__(self):
-        return ["username", "name", "organization", "email"]
+        return ["username"]
 
     @transaction.atomic
     def delete_related_data(self, delete_self=False):

checks/probes.py

@@ -1,7 +1,7 @@
 # Copyright: 2022, ECP, NLnet Labs and the Internet.nl contributors
 # SPDX-License-Identifier: Apache-2.0
 from django.conf import settings
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 
 from checks import categories
 from checks.categories import MailTlsStarttlsExists, WebTlsHttpsExists

checks/tasks/__init__.py

@@ -38,10 +38,11 @@ def ub_ctx(self):
                 self._ub_ctx.zone_add("test.", "transparent")
 
             self._ub_ctx.set_option("cache-max-ttl:", str(settings.CACHE_TTL * 0.9))
-            # Some (unknown) tests probably depend on consistent ordering in unbound responses
+            self._ub_ctx.set_option("cache-max-negative-ttl:", str(settings.CACHE_TTL * 0.9))
+            # Some may depend on consistent ordering in unbound responses
             # https://github.com/internetstandards/Internet.nl/pull/613#discussion_r892196819
+            # https://github.com/internetstandards/Internet.nl/pull/1292#discussion_r1505778673
             self._ub_ctx.set_option("rrset-roundrobin:", "no")
-            self._ub_ctx.set_option("cache-max-ttl:", str(settings.CACHE_TTL * 0.9))
             # XXX: Remove for now; inconsistency with applying settings on celery.
             # YYY: Removal caused infinite waiting on pipe to unbound. Added again.
             self._ub_ctx.set_async(True)

checks/tasks/ipv6.py

@@ -262,6 +262,7 @@ def callback(results, addr, parent, parent_name, category):
         elif testname == "web":
             parent.web_simhash_score = result.get("simhash_score")
             web_simhash_distance = result.get("simhash_distance")
+            simhash_status_code = result.get("simhash_status_code")
             parent.web_simhash_distance = web_simhash_distance
             parent.web_score = result.get("score")
 
@@ -276,8 +277,11 @@ def callback(results, addr, parent, parent_name, category):
                     category.subtests["web_reach"].result_good()
 
                 if len(good_conn) > 0:
-                    if web_simhash_distance <= settings.SIMHASH_MAX and web_simhash_distance >= 0:
-                        category.subtests["web_ipv46"].result_good()
+                    if settings.SIMHASH_MAX >= web_simhash_distance >= 0:
+                        if 200 <= simhash_status_code <= 400:
+                            category.subtests["web_ipv46"].result_good()
+                        else:
+                            category.subtests["web_ipv46"].result_notice_status_code()
                     elif web_simhash_distance == SIMHASH_NOT_CALCULABLE:
                         category.subtests["web_ipv46"].result_bad()
                     elif web_simhash_distance >= 0:
@@ -584,18 +588,23 @@ def strip_irrelevant_html(html):
             # Could not connect on given port, try another port.
             # If we managed to connect on IPv4 however, fail the test.
             if v4_response:
-                return simhash_score, distance
+                return simhash_score, distance, None
 
-    if not v4_response:
+    if v4_response is None:
         # FAIL: Could not establish a connection on both addresses.
-        return simhash_score, distance
+        return simhash_score, distance, v6_response.status_code
+
+    # Regardless of content, status code must be identical (#1267)
+    if v4_response.status_code != v6_response.status_code:
+        # FAIL: Could not establish a connection on both addresses.
+        return scoring.WEB_IPV6_WS_SIMHASH_OK, distance, v6_response.status_code
 
     try:
         html_v4 = response_content_chunk(v4_response, SIMHASH_MAX_RESPONSE_SIZE)
         html_v6 = response_content_chunk(v6_response, SIMHASH_MAX_RESPONSE_SIZE)
     except (OSError, IOError) as exc:
         log.debug("simhash encountered exception while reading response: {exc}", exc_info=exc)
-        return simhash_score, distance
+        return simhash_score, distance, v6_response.status_code
 
     for html, response in (html_v4, v4_response), (html_v6, v6_response):
         content_length = response.headers.get("content-length", "")
@@ -609,7 +618,7 @@ def strip_irrelevant_html(html):
     if distance <= settings.SIMHASH_MAX:
         simhash_score = scoring.WEB_IPV6_WS_SIMHASH_OK
 
-    return simhash_score, distance
+    return simhash_score, distance, v6_response.status_code
 
 
 def do_web(self, url, *args, **kwargs):
@@ -618,6 +627,8 @@ def do_web(self, url, *args, **kwargs):
         domain = []
         simhash_score = scoring.WEB_IPV6_WS_SIMHASH_FAIL
         simhash_distance = SIMHASH_NOT_CALCULABLE
+        simhash_status_code = None
+
         score = scoring.WEB_IPV6_WS_CONN_FAIL
 
         domain = get_domain_results(
@@ -641,7 +652,7 @@ def do_web(self, url, *args, **kwargs):
             simhash_score = scoring.WEB_IPV6_WS_SIMHASH_OK
             simhash_distance = -1
         elif len(v6_good) > 0 and len(v4_good) > 0 and len(v6_conn_diff) == 0:
-            simhash_score, simhash_distance = simhash(url, task=self)
+            simhash_score, simhash_distance, simhash_status_code = simhash(url, task=self)
 
     except SoftTimeLimitExceeded:
         log.debug("Soft time limit exceeded.")
@@ -656,4 +667,13 @@ def do_web(self, url, *args, **kwargs):
                 score=scoring.WEB_IPV6_WS_CONN_FAIL,
             )
 
-    return ("web", dict(domains=[domain], simhash_score=simhash_score, simhash_distance=simhash_distance, score=score))
+    return (
+        "web",
+        dict(
+            domains=[domain],
+            simhash_score=simhash_score,
+            simhash_distance=simhash_distance,
+            simhash_status_code=simhash_status_code,
+            score=score,
+        ),
+    )